chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,6 @@
|
||||
[bumpversion]
|
||||
current_version = 0.0.0
|
||||
commit = True
|
||||
tag = True
|
||||
|
||||
[bumpversion:file:pyproject.toml]
|
||||
@@ -0,0 +1,21 @@
|
||||
module.exports = {
|
||||
extends: ["@commitlint/config-conventional"],
|
||||
rules: {
|
||||
// Configuration Format: [level, applicability, value]
|
||||
// level: Error level, usually expressed as a number:
|
||||
// 0 - disable rule
|
||||
// 1 - Warning (does not prevent commits)
|
||||
// 2 - Error (will block the commit)
|
||||
// applicability: the conditions under which the rule applies, commonly used values:
|
||||
// “always” - always apply the rule
|
||||
// “never” - never apply the rule
|
||||
// value: the specific value of the rule, e.g. a maximum length of 100.
|
||||
// Refs: https://commitlint.js.org/reference/rules-configuration.html
|
||||
"header-max-length": [2, "always", 100],
|
||||
"type-enum": [
|
||||
2,
|
||||
"always",
|
||||
["build", "chore", "ci", "docs", "feat", "fix", "perf", "refactor", "revert", "style", "test", "Release-As"]
|
||||
]
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,10 @@
|
||||
# 1. Pull down your Azure Container Registry image
|
||||
FROM rdagentappregistry.azurecr.io/rd-agent-mle:20250623
|
||||
|
||||
# 2. (Optional) install any additional tools you need
|
||||
# e.g. git, bash-completion, etc.
|
||||
# RUN apt update && \
|
||||
# apt install -y git bash-completion && \
|
||||
# rm -rf /var/lib/apt/lists/*
|
||||
RUN apt update && \
|
||||
apt install -y git bash-completion
|
||||
@@ -0,0 +1,39 @@
|
||||
# Introduction
|
||||
|
||||
!!!!!This dev container is not for public development!!!!!!
|
||||
!!!!!Please don't use it if you are just a public open-source user.!!!!!!
|
||||
|
||||
# Steps to run the dev container (for internal use only)
|
||||
|
||||
Prerequisites(this is the reason why this dev container is not for public use):
|
||||
|
||||
- Make sure you have the `rdagentappregistry.azurecr.io/rd-agent-mle:20250623` image locally & DevContainer is installed in your IDE
|
||||
- The kaggle dataset is located at `/home/shared/RD-Agent/kaggle`
|
||||
|
||||
1. Open the project and select "Open In DevContainer"
|
||||
2. Set up your Kaggle Key (do not share this; other internal URLs are hardcoded in the config files)
|
||||
|
||||
```bash
|
||||
export KAGGLE_USERNAME=
|
||||
export KAGGLE_KEY=
|
||||
```
|
||||
|
||||
3. Run: python rdagent/app/data_science/loop.py --competition nomad2018-predict-transparent-conductors
|
||||
|
||||
|
||||
# Additional Notes
|
||||
- Please install and use this Dev Container in VS Code.
|
||||
- You **must open VS Code remotely and enter the `RD-Agent` directory before running the DevContainer configuration (`.devcontainer/devcontainer.json`)**. Otherwise, the workspace and path mappings will not work as expected.
|
||||
- To open the DevContainer correctly in VS Code:
|
||||
1. Remotely connect to the machine and open the `RD-Agent` folder in VS Code.
|
||||
2. Press `Ctrl+Shift+P` (or `Cmd+Shift+P` on Mac), type and select **"Dev Containers: Reopen in Container"**.
|
||||
|
||||
|
||||
|
||||
# How to grade your submission in the DevContainer
|
||||
|
||||
1. save your submission file in `./sumission.csv`
|
||||
|
||||
2. Run evaluation
|
||||
DS_COMPETITION=<your competition name>
|
||||
conda run -n mlebench mlebench grade-sample submission.csv $DS_COMPETITION --data-dir /tmp/kaggle/zip_files/
|
||||
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"name": "rd-agent-mle DevContainer",
|
||||
"build": {
|
||||
"dockerfile": "Dockerfile",
|
||||
"context": ".."
|
||||
},
|
||||
"workspaceFolder": "/workspace/RD-Agent",
|
||||
"workspaceMount": "source=${localWorkspaceFolder},target=/workspace/RD-Agent,type=bind,consistency=cached",
|
||||
"remoteUser": "root",
|
||||
"settings": {
|
||||
"terminal.integrated.shell.linux": "/bin/bash"
|
||||
},
|
||||
"mounts": [
|
||||
"source=/home/shared/RD-Agent/kaggle,target=/tmp/kaggle,type=bind,consistency=cached,readonly"
|
||||
],
|
||||
"extensions": [
|
||||
"ms-python.python",
|
||||
"ms-python.vscode-pylance",
|
||||
"ms-toolsai.jupyter"
|
||||
],
|
||||
"runArgs": [
|
||||
"--init",
|
||||
"--shm-size=1g",
|
||||
"--env-file", "${localWorkspaceFolder}/.devcontainer/env",
|
||||
"--network=host",
|
||||
"--gpus=all"
|
||||
],
|
||||
"postCreateCommand": "make dev"
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
# Global configs:
|
||||
|
||||
MAX_RETRY=12000
|
||||
RETRY_WAIT_SECONDS=5
|
||||
TIMEOUT_FAIL_LIMIT=100
|
||||
|
||||
# litellm
|
||||
# CHAT_MODEL=gpt-4o
|
||||
# CHAT_TEMPERATURE=0.7
|
||||
|
||||
CHAT_STREAM=False
|
||||
CHAT_TEMPERATURE=1
|
||||
CHAT_MODEL=o1-preview
|
||||
SYSTEM_PROMPT_ROLE=user
|
||||
|
||||
BACKEND=rdagent.oai.backend.LiteLLMAPIBackend
|
||||
OPENAI_API_KEY=sk-1234
|
||||
OPENAI_API_BASE=http://ep14.213428.xyz:38881
|
||||
|
||||
|
||||
# amc chat model configs:
|
||||
EMBEDDING_MODEL=text-embedding-ada-002
|
||||
|
||||
# Cache Setting (Optional):
|
||||
DUMP_CHAT_CACHE=True
|
||||
USE_CHAT_CACHE=False
|
||||
DUMP_EMBEDDING_CACHE=True
|
||||
USE_EMBEDDING_CACHE=False
|
||||
LOG_LLM_CHAT_CONTENT=True
|
||||
|
||||
DS_LOCAL_DATA_PATH=/tmp/kaggle
|
||||
|
||||
DS_IF_USING_MLE_DATA=True
|
||||
|
||||
|
||||
PICKLE_CACHE_FOLDER_PATH_STR=./log/pickle_cache
|
||||
CACHE_WITH_PICKLE=False
|
||||
ENABLE_CACHE=False
|
||||
PROMPT_CACHE_PATH=./log/prompt_cache.db
|
||||
|
||||
DS_CODER_COSTEER_ENV_TYPE=conda
|
||||
# DS_PROPOSAL_VERSION=v2 deprecated
|
||||
|
||||
DS_CODER_ON_WHOLE_PIPELINE=True
|
||||
COSTEER_V2_QUERY_FORMER_TRACE_LIMIT=3
|
||||
|
||||
# export PYTHONPATH=. # this is for running researcher branch;
|
||||
@@ -0,0 +1,2 @@
|
||||
github:
|
||||
- MIIC-finance
|
||||
@@ -0,0 +1,51 @@
|
||||
---
|
||||
name: "\U0001F41B Bug Report"
|
||||
about: Submit a bug report to help us improve RD-Agent
|
||||
labels: bug
|
||||
|
||||
---
|
||||
|
||||
## 🐛 Bug Description
|
||||
|
||||
<!-- A clear and concise description of what the bug is. -->
|
||||
|
||||
## To Reproduce
|
||||
|
||||
Steps to reproduce the behavior:
|
||||
|
||||
1.
|
||||
2.
|
||||
3.
|
||||
|
||||
|
||||
## Expected Behavior
|
||||
|
||||
<!-- A clear and concise description of what you expected to happen. -->
|
||||
|
||||
## Screenshot
|
||||
|
||||
<!-- A screenshot of the error message or anything shouldn't appear-->
|
||||
|
||||
## Environment
|
||||
|
||||
**Note**: Users can run `rdagent collect_info` to get system information and paste it directly here.
|
||||
|
||||
- Name of current operating system:
|
||||
- Processor architecture:
|
||||
- System, version, and hardware information:
|
||||
- Version number of the system:
|
||||
- Python version:
|
||||
- Container ID:
|
||||
- Container Name:
|
||||
- Container Status:
|
||||
- Image ID used by the container:
|
||||
- Image tag used by the container:
|
||||
- Container port mapping:
|
||||
- Container Label:
|
||||
- Startup Commands:
|
||||
- RD-Agent version:
|
||||
- Package version:
|
||||
|
||||
## Additional Notes
|
||||
|
||||
<!-- Add any other information about the problem here. -->
|
||||
@@ -0,0 +1,9 @@
|
||||
---
|
||||
name: "\U0001F4D6 Documentation"
|
||||
about: Report an issue related to documentation
|
||||
|
||||
---
|
||||
|
||||
## 📖 Documentation
|
||||
|
||||
<!-- Please specify whether it's tutorial part or API reference part, and describe it.-->
|
||||
@@ -0,0 +1,25 @@
|
||||
---
|
||||
name: "\U0001F31FFeature Request"
|
||||
about: Request for a new RD-Agent feature
|
||||
labels: enhancement
|
||||
|
||||
---
|
||||
|
||||
## 🌟 Feature Description
|
||||
<!-- A clear and concise description of the feature proposal -->
|
||||
|
||||
## Motivation
|
||||
|
||||
1. Application scenario
|
||||
2. Related works (Papers, Github repos etc.):
|
||||
3. Any other relevant and important information:
|
||||
|
||||
<!-- Please describe why the feature is important. -->
|
||||
|
||||
## Alternatives
|
||||
|
||||
<!-- A short description of any alternative solutions or features you've considered. -->
|
||||
|
||||
## Additional Notes
|
||||
|
||||
<!-- Add any other context or screenshots about the feature request here. -->
|
||||
@@ -0,0 +1,10 @@
|
||||
---
|
||||
name: "❓Questions & Help"
|
||||
about: Have some questions? We can offer help.
|
||||
labels: question
|
||||
|
||||
---
|
||||
|
||||
## ❓ Questions and Help
|
||||
|
||||
We sincerely suggest you to carefully read the [documentation](http://rdagent.readthedocs.io/). After that, if you still feel puzzled, please describe the question clearly under this issue.
|
||||
@@ -0,0 +1,34 @@
|
||||
<!--- Thank you for submitting a Pull Request! In order to make our work smoother. -->
|
||||
<!--- please make sure your Pull Request meets the following requirements: -->
|
||||
<!--- 1. Provide a general summary of your changes in the Title above; -->
|
||||
<!--- 2. Add appropriate prefixes to titles, such as `build:`, `chore:`, `ci:`, `docs:`, `feat:`, `fix:`, `perf:`, `refactor:`, `revert:`, `style:`, `test:`(Ref: https://www.conventionalcommits.org/). -->
|
||||
<!--- Category: -->
|
||||
<!--- Patch Updates: `fix:` -->
|
||||
<!--- Example: fix(auth): correct login validation issue -->
|
||||
<!--- minor update (introduces new functionality): `feat` -->
|
||||
<!--- Example: feature(parser): add ability to parse arrays -->
|
||||
<!--- major update(destructive update): Include BREAKING CHANGE in the commit message footer, or add `! ` in the commit footer to indicate that there is a destructive update. -->
|
||||
<!--- Example: feat(auth)! : remove support for old authentication method -->
|
||||
<!--- Other updates: `build:`, `chore:`, `ci:`, `docs:`, `perf:`, `refactor:`, `revert:`, `style:`, `test:`. -->
|
||||
|
||||
## Description
|
||||
<!--- Describe your changes in detail -->
|
||||
|
||||
## Motivation and Context
|
||||
<!--- Are there any related issues? If so, please put the link here. -->
|
||||
<!--- Why is this change required? What problem does it solve? -->
|
||||
|
||||
## How Has This Been Tested?
|
||||
<!--- Put an `x` in all the boxes that apply: --->
|
||||
- [ ] If you are adding a new feature, test on your own test scripts.
|
||||
|
||||
<!--- **ATTENTION**: If you are adding a new feature, please make sure your codes are **correctly tested**. If our test scripts do not cover your cases, please provide your own test scripts under the `tests` folder and test them. More information about test scripts can be found [here](https://docs.python.org/3/library/unittest.html#basic-example), or you could refer to those we provide under the `tests` folder. -->
|
||||
|
||||
## Screenshots of Test Results (if appropriate):
|
||||
1. Your own tests:
|
||||
|
||||
## Types of changes
|
||||
<!--- What types of changes does your code introduce? Put an `x` in all the boxes that apply: -->
|
||||
- [ ] Fix bugs
|
||||
- [ ] Add new feature
|
||||
- [ ] Update documentation
|
||||
@@ -0,0 +1,19 @@
|
||||
updates:
|
||||
- commit-message:
|
||||
prefix: build(actions)
|
||||
directory: /
|
||||
package-ecosystem: github-actions
|
||||
schedule:
|
||||
interval: weekly
|
||||
- commit-message:
|
||||
prefix: build(requirements)
|
||||
directory: /
|
||||
groups:
|
||||
dev:
|
||||
dependency-type: development
|
||||
prod:
|
||||
dependency-type: production
|
||||
package-ecosystem: pip
|
||||
schedule:
|
||||
interval: weekly
|
||||
version: 2
|
||||
@@ -0,0 +1,69 @@
|
||||
concurrency:
|
||||
cancel-in-progress: true
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
jobs:
|
||||
ci:
|
||||
if: ${{ !cancelled() && ! failure() }}
|
||||
needs: dependabot
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
submodules: recursive
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
cache: pip
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- run: make dev
|
||||
- name: lint test docs and build
|
||||
run: make lint docs-gen test-offline # test docs build
|
||||
strategy:
|
||||
matrix:
|
||||
python-version:
|
||||
- '3.10'
|
||||
- '3.11'
|
||||
dependabot:
|
||||
if: ${{ github.actor == 'dependabot[bot]' && startsWith(github.head_ref, 'dependabot/pip/') }}
|
||||
permissions:
|
||||
contents: write
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
ref: ${{ github.head_ref }}
|
||||
- name: Set up Git
|
||||
run: |
|
||||
git config --global user.name github-actions
|
||||
git config --global user.email github-actions@github.com
|
||||
- name: Set up Python with multiple versions.
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
cache: pip
|
||||
python-version: |
|
||||
3.10
|
||||
3.11
|
||||
- name: Install pipenv using pipx
|
||||
run: pipx install pipenv
|
||||
- name: Generate constraints for all supported Python versions
|
||||
run: |
|
||||
CI= PYTHON_VERSION=3.10 make constraints
|
||||
CI= PYTHON_VERSION=3.11 make constraints
|
||||
- name: Push changes if applicable
|
||||
run: |
|
||||
if [[ -n `git status --porcelain` ]]; then
|
||||
git commit -a -m "build: Update constraints for dependabot."
|
||||
git push
|
||||
fi
|
||||
name: CI
|
||||
on:
|
||||
pull_request:
|
||||
types:
|
||||
- opened
|
||||
- synchronize
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
@@ -0,0 +1,35 @@
|
||||
name: Lint pull request title
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types:
|
||||
- opened
|
||||
- synchronize
|
||||
- reopened
|
||||
- edited
|
||||
|
||||
concurrency:
|
||||
cancel-in-progress: true
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
|
||||
jobs:
|
||||
lint-title:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
# This step is necessary because the lint title uses the .commitlintrc.js file in the project root directory.
|
||||
- name: Checkout Repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '16'
|
||||
|
||||
- name: Install commitlint
|
||||
run: npm install --save-dev @commitlint/{config-conventional,cli}
|
||||
|
||||
- name: Validate PR Title with commitlint
|
||||
env:
|
||||
BODY: ${{ github.event.pull_request.title }}
|
||||
run: |
|
||||
echo "$BODY" | npx commitlint --config .commitlintrc.js
|
||||
@@ -0,0 +1,17 @@
|
||||
concurrency:
|
||||
cancel-in-progress: true
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
jobs:
|
||||
documentation-links:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: readthedocs/actions/preview@v1
|
||||
with:
|
||||
project-slug: RDAgent
|
||||
name: Read the Docs Pull Request Preview
|
||||
on:
|
||||
pull_request_target:
|
||||
types:
|
||||
- opened
|
||||
permissions:
|
||||
pull-requests: write
|
||||
@@ -0,0 +1,48 @@
|
||||
name: Release
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
permissions:
|
||||
contents: read
|
||||
jobs:
|
||||
release_and_publish:
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: read
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Release please
|
||||
id: release_please
|
||||
uses: googleapis/release-please-action@v4
|
||||
with:
|
||||
# The current PAT (personal access token) was created on 2024-08-05,
|
||||
# since the maximum validity of PAT is 1 year, you need to change the PAT before 2025-08-05.
|
||||
token: ${{ secrets.PAT }}
|
||||
release-type: simple
|
||||
- uses: actions/checkout@v4
|
||||
if: ${{ steps.release_please.outputs.release_created }}
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Set up Python
|
||||
if: ${{ steps.release_please.outputs.release_created }}
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
cache: pip
|
||||
python-version: '3.10'
|
||||
- name: Install dependencies
|
||||
if: ${{ steps.release_please.outputs.release_created }}
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install setuptools wheel twine # better-exceptions(optional for debug)
|
||||
- run: make dev
|
||||
if: ${{ steps.release_please.outputs.release_created }}
|
||||
- run: make build
|
||||
if: ${{ steps.release_please.outputs.release_created }}
|
||||
- name: upload
|
||||
if: ${{ steps.release_please.outputs.release_created }}
|
||||
env:
|
||||
TWINE_USERNAME: __token__
|
||||
TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
|
||||
run: |
|
||||
make upload
|
||||
+190
@@ -0,0 +1,190 @@
|
||||
# Custom
|
||||
*.swp
|
||||
.DS_Store
|
||||
Pipfile
|
||||
public
|
||||
release-notes.md
|
||||
typescript*
|
||||
tmp/
|
||||
.ai/
|
||||
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
pip-wheel-metadata/
|
||||
share/python-wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
*.py,cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
/log*/
|
||||
local_settings.py
|
||||
db.sqlite3
|
||||
db.sqlite3-journal
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
|
||||
# PyBuilder
|
||||
target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
|
||||
# IPython
|
||||
profile_default/
|
||||
ipython_config.py
|
||||
|
||||
# pyenv
|
||||
.python-version
|
||||
|
||||
# pipenv
|
||||
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||
# install all needed dependencies.
|
||||
#Pipfile.lock
|
||||
|
||||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
|
||||
__pypackages__/
|
||||
|
||||
# Celery stuff
|
||||
celerybeat-schedule
|
||||
celerybeat.pid
|
||||
|
||||
# SageMath parsed files
|
||||
*.sage.py
|
||||
|
||||
# Environments
|
||||
.env*
|
||||
*.env
|
||||
.venv
|
||||
^env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
|
||||
# mkdocs documentation
|
||||
/site
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
|
||||
# Pyre type checker
|
||||
.pyre/
|
||||
|
||||
# all pkl files
|
||||
*.pkl
|
||||
|
||||
# all h5 files
|
||||
*.h5
|
||||
|
||||
# all vs-code files
|
||||
.vscode/
|
||||
|
||||
# reports
|
||||
reports/
|
||||
|
||||
# git_ignore_folder
|
||||
git_ignore_folder/
|
||||
|
||||
#cache
|
||||
*cache*/
|
||||
*cache.json
|
||||
|
||||
# DB files
|
||||
*.db
|
||||
|
||||
# Docker
|
||||
factor_template/mlruns/
|
||||
env_tpl
|
||||
mlruns/
|
||||
|
||||
# possible output from coder or runner
|
||||
*.pth
|
||||
*qlib_res.csv
|
||||
|
||||
# shell script
|
||||
*.out
|
||||
/*.sh
|
||||
.aider*
|
||||
rdagent/app/benchmark/factor/example.json
|
||||
|
||||
# UI Server resources
|
||||
videos/
|
||||
static/
|
||||
|
||||
# AI assistant
|
||||
.cursor/
|
||||
.claude/
|
||||
AGENTS.md
|
||||
!rdagent/**/AGENTS.md
|
||||
|
||||
scripts/
|
||||
@@ -0,0 +1,38 @@
|
||||
# .readthedocs.yml
|
||||
# Read the Docs configuration file
|
||||
# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
|
||||
|
||||
# Required
|
||||
version: 2
|
||||
|
||||
# Set the version of Python and other tools you might need
|
||||
build:
|
||||
os: ubuntu-22.04
|
||||
tools:
|
||||
python: "3.10"
|
||||
# During the build process, you need to fetch tags, and since the default command to read the docs only pulls shallow code, it will cause an error.
|
||||
# So we added the `git fetch --tags --unshallow || true` command to fetch the full tag record.
|
||||
# Adding this command overrides the default command, so we copied it over to make sure the build was successful.
|
||||
commands:
|
||||
- python -mvirtualenv $READTHEDOCS_VIRTUALENV_PATH
|
||||
- python -m pip install --upgrade --no-cache-dir pip setuptools
|
||||
- python -m pip install --upgrade --no-cache-dir sphinx
|
||||
- python -m pip install --exists-action=w --no-cache-dir -r requirements/docs.txt
|
||||
- python -m pip install --upgrade --upgrade-strategy only-if-needed --no-cache-dir .
|
||||
- git fetch --tags --unshallow || true
|
||||
- mkdir -p $READTHEDOCS_OUTPUT/html/
|
||||
- python -m sphinx -T -b html -d _build/doctrees -D language=en ./docs $READTHEDOCS_OUTPUT/html
|
||||
|
||||
# Build documentation in the docs/ directory with Sphinx
|
||||
sphinx:
|
||||
configuration: docs/conf.py
|
||||
|
||||
# Build all formats
|
||||
formats: all
|
||||
|
||||
# Optionally set the version of Python and requirements required to build your docs
|
||||
python:
|
||||
install:
|
||||
- requirements: requirements/docs.txt
|
||||
- method: pip
|
||||
path: .
|
||||
@@ -0,0 +1,2 @@
|
||||
[client]
|
||||
showSidebarNavigation = false
|
||||
+592
@@ -0,0 +1,592 @@
|
||||
# Changelog
|
||||
|
||||
## [0.8.0](https://github.com/microsoft/RD-Agent/compare/v0.7.0...v0.8.0) (2025-11-03)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* add a rag mcp in proposal ([#1267](https://github.com/microsoft/RD-Agent/issues/1267)) ([a0cd102](https://github.com/microsoft/RD-Agent/commit/a0cd1025c141aee6d4e6cb10286c77d827b89379))
|
||||
* add coder check and give more time ([#1127](https://github.com/microsoft/RD-Agent/issues/1127)) ([e32d229](https://github.com/microsoft/RD-Agent/commit/e32d229f2b722acac53f4e2f7d8a98e29cb19dc1))
|
||||
* add enable_cache toggle for UI data caching ([#1075](https://github.com/microsoft/RD-Agent/issues/1075)) ([0c9f193](https://github.com/microsoft/RD-Agent/commit/0c9f1930e8d5df1c00bfb32ee578da2dc53db1ec))
|
||||
* add extra_eval config and import_class for custom evaluators ([#1097](https://github.com/microsoft/RD-Agent/issues/1097)) ([5accec3](https://github.com/microsoft/RD-Agent/commit/5accec37c8828ac42005c2d12b815bef599b547e))
|
||||
* add hypo_critic and hypo_rewrite in proposal ([#1106](https://github.com/microsoft/RD-Agent/issues/1106)) ([71440f6](https://github.com/microsoft/RD-Agent/commit/71440f643fc9d952dfa064359c1945b729dbfd9f))
|
||||
* add improve_mode to MultiProcessEvolvingStrategy for selective task implementation ([#1273](https://github.com/microsoft/RD-Agent/issues/1273)) ([9344635](https://github.com/microsoft/RD-Agent/commit/93446356952803d8b1f1eb0c39da825c19274cb6))
|
||||
* add loop ID mapping to trace nodes and update UI labels ([#1098](https://github.com/microsoft/RD-Agent/issues/1098)) ([5437851](https://github.com/microsoft/RD-Agent/commit/54378518dadd6c38496eceda8ef5b33b375a5c97))
|
||||
* add mask inference in debug mode ([#1154](https://github.com/microsoft/RD-Agent/issues/1154)) ([ef749ab](https://github.com/microsoft/RD-Agent/commit/ef749ab744fb6fbafd1a8e6a3642cce20ce96069))
|
||||
* add only success filter toggle for traces ([#1047](https://github.com/microsoft/RD-Agent/issues/1047)) ([5e582cc](https://github.com/microsoft/RD-Agent/commit/5e582cc71d5c153666c465cb2d797dc71e43c501))
|
||||
* add option to enable hyperparameter tuning only in first eval loop ([#1211](https://github.com/microsoft/RD-Agent/issues/1211)) ([bc3fa17](https://github.com/microsoft/RD-Agent/commit/bc3fa170b029f50c8f7b1828cdf4ffd024e64b8b))
|
||||
* add previous runner loops to runner history ([#1142](https://github.com/microsoft/RD-Agent/issues/1142)) ([8de9f75](https://github.com/microsoft/RD-Agent/commit/8de9f757ea134b04cde0622c6225678d85a87862))
|
||||
* add reasoning attribute to DSRunnerFeedback for enhanced evaluation context ([#1162](https://github.com/microsoft/RD-Agent/issues/1162)) ([4e41c97](https://github.com/microsoft/RD-Agent/commit/4e41c9797cbafd35cc0d883fede4226398c573e1))
|
||||
* add sample submission file check ([#1053](https://github.com/microsoft/RD-Agent/issues/1053)) ([6a840d8](https://github.com/microsoft/RD-Agent/commit/6a840d819251e64d98daa40289592a05ac5fb369))
|
||||
* add show_hard_limit option and update time limit handling in DataScience settings ([#1144](https://github.com/microsoft/RD-Agent/issues/1144)) ([fe762cd](https://github.com/microsoft/RD-Agent/commit/fe762cd860a109b426e3d89a6fbc3c161d77b5e2))
|
||||
* add stdout into workspace for easier debugging ([#1236](https://github.com/microsoft/RD-Agent/issues/1236)) ([d3d4967](https://github.com/microsoft/RD-Agent/commit/d3d4967a129ad986d5087add4b101d913e1e14ba))
|
||||
* add time ratio limit for hyperparameter tuning in Kaggle settin… ([#1135](https://github.com/microsoft/RD-Agent/issues/1135)) ([e44bc83](https://github.com/microsoft/RD-Agent/commit/e44bc8356a93b63eb120e88336eaf4c5b05ccd97))
|
||||
* add user interaction in data science scenario ([#1251](https://github.com/microsoft/RD-Agent/issues/1251)) ([2afef70](https://github.com/microsoft/RD-Agent/commit/2afef703ca0e670197d02aab7f9c4f6e3e409872))
|
||||
* add ws CLI and support optional timeout/cache ([#1066](https://github.com/microsoft/RD-Agent/issues/1066)) ([fae3def](https://github.com/microsoft/RD-Agent/commit/fae3defefa38e91131d4e351d68f4484ca280956))
|
||||
* analyze feedback based on sota numbers ([#1116](https://github.com/microsoft/RD-Agent/issues/1116)) ([167f5e2](https://github.com/microsoft/RD-Agent/commit/167f5e2fe9a5679d5beca2f7d3093ac0fd17e664))
|
||||
* create Jupyter notebook pipeline file based on main.py file ([#1134](https://github.com/microsoft/RD-Agent/issues/1134)) ([2fa1790](https://github.com/microsoft/RD-Agent/commit/2fa1790cb3852d96a197fd7970af4063339dfa26))
|
||||
* enable drafting with knowledge ([#998](https://github.com/microsoft/RD-Agent/issues/998)) ([8e385eb](https://github.com/microsoft/RD-Agent/commit/8e385ebf422256d08f02c055ab64115872b69d94))
|
||||
* enable finetune llm ([#1055](https://github.com/microsoft/RD-Agent/issues/1055)) ([909c7d6](https://github.com/microsoft/RD-Agent/commit/909c7d6e8a35ce8c43d29201eccfe5cd2a21049d))
|
||||
* enable LLM‑based hypothesis selection with time‑aware prompt & colored logging ([#1122](https://github.com/microsoft/RD-Agent/issues/1122)) ([1c4ab89](https://github.com/microsoft/RD-Agent/commit/1c4ab89f52fbdff7cab68ee1b778703b20514a9b))
|
||||
* enable meta planner ([#1103](https://github.com/microsoft/RD-Agent/issues/1103)) ([c208209](https://github.com/microsoft/RD-Agent/commit/c20820929b7fcdd5c9fbb81e63bad0ba76239c50))
|
||||
* enable to inject diversity cross async multi-trace ([#1173](https://github.com/microsoft/RD-Agent/issues/1173)) ([bcdd957](https://github.com/microsoft/RD-Agent/commit/bcdd957c71b59d8664ecb1523b5fcf2179aa1138))
|
||||
* enhance timeout handling in CoSTEER and DataScience scenarios ([#1150](https://github.com/microsoft/RD-Agent/issues/1150)) ([06233cb](https://github.com/microsoft/RD-Agent/commit/06233cb95acb1df01ca71b1a554cf4a5f2c4d092))
|
||||
* enhance timeout management and knowledge base handling in CoSTEER components ([#1130](https://github.com/microsoft/RD-Agent/issues/1130)) ([963d260](https://github.com/microsoft/RD-Agent/commit/963d26001e346c05bcc540536f65d9a199ca6ac5))
|
||||
* fallback to acceptable results ([#1129](https://github.com/microsoft/RD-Agent/issues/1129)) ([3ce2bd4](https://github.com/microsoft/RD-Agent/commit/3ce2bd41c442c6b756810c7895b1e6a1df13dfbb))
|
||||
* improve fallback handling in CoSTEER and add GPU usage guidelin… ([#1165](https://github.com/microsoft/RD-Agent/issues/1165)) ([cec4240](https://github.com/microsoft/RD-Agent/commit/cec424046759f02735a6b49e3a9f615a403b62c9))
|
||||
* init pydantic ai agent & context 7 mcp ([#1240](https://github.com/microsoft/RD-Agent/issues/1240)) ([59af538](https://github.com/microsoft/RD-Agent/commit/59af5383d7d1d73a5e3630da9d1bbfed31111436))
|
||||
* **mcp:** cache with one-click toggle ([#1269](https://github.com/microsoft/RD-Agent/issues/1269)) ([6f86863](https://github.com/microsoft/RD-Agent/commit/6f86863b63ae331f9b7761eaf9ae0a85aca7ba42))
|
||||
* mcts policy based on trace scheduler ([#1203](https://github.com/microsoft/RD-Agent/issues/1203)) ([13890e0](https://github.com/microsoft/RD-Agent/commit/13890e0bbcaf5a7a87a7bff55e720b0c3bbbbfe9))
|
||||
* new prompt for auto-sota-selector ([#1109](https://github.com/microsoft/RD-Agent/issues/1109)) ([13c92a9](https://github.com/microsoft/RD-Agent/commit/13c92a90eee275e40a9a2fb0b853c8ecb2bd59fd))
|
||||
* offline selector ([#1231](https://github.com/microsoft/RD-Agent/issues/1231)) ([76b2e87](https://github.com/microsoft/RD-Agent/commit/76b2e87348cbeb983606691fdf343c4fc721c2bb))
|
||||
* prob-based trace scheduler ([#1131](https://github.com/microsoft/RD-Agent/issues/1131)) ([970561a](https://github.com/microsoft/RD-Agent/commit/970561a057ed5e56e29be3577b7c062aca4b49b6))
|
||||
* query & cache package_info ([#1083](https://github.com/microsoft/RD-Agent/issues/1083)) ([19869ea](https://github.com/microsoft/RD-Agent/commit/19869ea4752b67b62ffdcb8d54632a59661b5466))
|
||||
* refactor CoSTEER classes to use DSCoSTEER and update max seconds handling ([#1156](https://github.com/microsoft/RD-Agent/issues/1156)) ([6d01e3e](https://github.com/microsoft/RD-Agent/commit/6d01e3e1ca1eec281b52f461724bf63adefe5d81))
|
||||
* refine the logic of enabling hyperparameter tuning and add criteira ([#1175](https://github.com/microsoft/RD-Agent/issues/1175)) ([af071f5](https://github.com/microsoft/RD-Agent/commit/af071f5f45bfeb524a0f16da84d802e523478213))
|
||||
* show the summarized final difference between the final workspace and the base workspace ([#1281](https://github.com/microsoft/RD-Agent/issues/1281)) ([2bf8345](https://github.com/microsoft/RD-Agent/commit/2bf83453921457e44c802913a8e24b0de98611bd))
|
||||
* streamline hyperparameter tuning checks and update evaluation g… ([#1167](https://github.com/microsoft/RD-Agent/issues/1167)) ([383e5ed](https://github.com/microsoft/RD-Agent/commit/383e5ed488c73abedb41acb2ea27afd60738669f))
|
||||
* ui, support disable cache ([#1217](https://github.com/microsoft/RD-Agent/issues/1217)) ([92efe33](https://github.com/microsoft/RD-Agent/commit/92efe33fa9c8be54a71bf0840f867edc877236fe))
|
||||
* update README with latest paper acceptance to NeurIPS 2025 ([#1252](https://github.com/microsoft/RD-Agent/issues/1252)) ([8332960](https://github.com/microsoft/RD-Agent/commit/833296084f3b3d0fea15fd693e302c26b2d80762))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* add a switch for ensemble_time_upper_bound and fix some bug in main ([#1226](https://github.com/microsoft/RD-Agent/issues/1226)) ([f00a538](https://github.com/microsoft/RD-Agent/commit/f00a5382b16379aaea2dfabf09a681be25e29d3e))
|
||||
* add gpu_info in research phase ([#1094](https://github.com/microsoft/RD-Agent/issues/1094)) ([58c9c1b](https://github.com/microsoft/RD-Agent/commit/58c9c1b9b62d6d25b9b6980e19959664ef7272d7))
|
||||
* add json format response fallback to prompt templates ([#1246](https://github.com/microsoft/RD-Agent/issues/1246)) ([4dfb8a1](https://github.com/microsoft/RD-Agent/commit/4dfb8a130b3970192d3a8da799152de492c79aec))
|
||||
* add metric in scores.csv and avoid reading sample_submission.csv ([#1152](https://github.com/microsoft/RD-Agent/issues/1152)) ([fd039f1](https://github.com/microsoft/RD-Agent/commit/fd039f1f8184c9107539f270735f227cf68c62c0))
|
||||
* add missing self parameter to instance methods in DSProposalV2ExpGen ([#1213](https://github.com/microsoft/RD-Agent/issues/1213)) ([68af035](https://github.com/microsoft/RD-Agent/commit/68af03517749cff4726acb016daad561148147bf))
|
||||
* add spec for hyperparameters in task design and coder ([#995](https://github.com/microsoft/RD-Agent/issues/995)) ([10246fd](https://github.com/microsoft/RD-Agent/commit/10246fd2491d48560d5f7055f78906e7a6a2882e))
|
||||
* align scenario descriptions and include debug timeout ([#1079](https://github.com/microsoft/RD-Agent/issues/1079)) ([13b6663](https://github.com/microsoft/RD-Agent/commit/13b66630ec17f1ed4f52a9d8ea0913722ca74483))
|
||||
* allow prev_out keys to be None in workspace cleanup assertion ([#1214](https://github.com/microsoft/RD-Agent/issues/1214)) ([1f4d190](https://github.com/microsoft/RD-Agent/commit/1f4d190a3209bbe4ec960f8dd79be59672cd0e7f))
|
||||
* based on response schema; not function calling ([#1038](https://github.com/microsoft/RD-Agent/issues/1038)) ([99da8c5](https://github.com/microsoft/RD-Agent/commit/99da8c58f0f779aa19edc2522d4cf143577811d8))
|
||||
* cancel tasks on resume and kill subprocesses on termination ([#1166](https://github.com/microsoft/RD-Agent/issues/1166)) ([cf6e418](https://github.com/microsoft/RD-Agent/commit/cf6e418eb8d899e22c93279055d42c185397fa2a))
|
||||
* change runner prompts ([#1223](https://github.com/microsoft/RD-Agent/issues/1223)) ([6d3e73d](https://github.com/microsoft/RD-Agent/commit/6d3e73d679a8ffe4a48923590a7c37b4fdcd207a))
|
||||
* clear ws_ckp after extraction to reduce workspace object size ([#1137](https://github.com/microsoft/RD-Agent/issues/1137)) ([783affe](https://github.com/microsoft/RD-Agent/commit/783affe0d513b2e9fbcbb11e0408cc79db19a274))
|
||||
* correct DS_LOCAL_DATA_PATH error in devcontainer ([#1063](https://github.com/microsoft/RD-Agent/issues/1063)) ([588fcfa](https://github.com/microsoft/RD-Agent/commit/588fcfa3ab0a4eca5afee766e3f56f094b28a999))
|
||||
* **dockerfile:** install coreutils to resolve timeout command error ([#1260](https://github.com/microsoft/RD-Agent/issues/1260)) ([07f89b0](https://github.com/microsoft/RD-Agent/commit/07f89b013ea99102f4875fda5704adde14cf9978))
|
||||
* **docs:** update rdagent ui with correct params ([#1249](https://github.com/microsoft/RD-Agent/issues/1249)) ([f360d0a](https://github.com/microsoft/RD-Agent/commit/f360d0a212793eb044c218b5e13b095e684a632d))
|
||||
* enable embedding truncation ([#1188](https://github.com/microsoft/RD-Agent/issues/1188)) ([2421fa4](https://github.com/microsoft/RD-Agent/commit/2421fa4493bd86c98ff672afc26ec71ba510e391))
|
||||
* enhance feedback handling in MultiProcessEvolvingStrategy for improved task evolution ([#1274](https://github.com/microsoft/RD-Agent/issues/1274)) ([961e561](https://github.com/microsoft/RD-Agent/commit/961e56102cddae3348af46a30f9085f353151890))
|
||||
* error in prompt template ([#1065](https://github.com/microsoft/RD-Agent/issues/1065)) ([a90e598](https://github.com/microsoft/RD-Agent/commit/a90e598e568c0339a5f29577fbf44e302bc0d96f))
|
||||
* filter log folders bug in ui ([#1073](https://github.com/microsoft/RD-Agent/issues/1073)) ([d0f33c5](https://github.com/microsoft/RD-Agent/commit/d0f33c56733bb28222c1f2c8f8a0ff5604ddf858))
|
||||
* fix a bug in return curve display ([#1042](https://github.com/microsoft/RD-Agent/issues/1042)) ([249f661](https://github.com/microsoft/RD-Agent/commit/249f6614a67d8b38e9ad2f0d95154db7071e8e3a))
|
||||
* fix a small bug in json_mode ([#1041](https://github.com/microsoft/RD-Agent/issues/1041)) ([8bc12ea](https://github.com/microsoft/RD-Agent/commit/8bc12eaaa7ecda69043ec781896299a6796c8140))
|
||||
* fix a small bug in response_schema ([#1043](https://github.com/microsoft/RD-Agent/issues/1043)) ([66cadcd](https://github.com/microsoft/RD-Agent/commit/66cadcd7b2a91bac416acd94196b96f43b572c2b))
|
||||
* fix bug for hypo_select_with_llm when not support response_schema ([#1208](https://github.com/microsoft/RD-Agent/issues/1208)) ([54cc2c4](https://github.com/microsoft/RD-Agent/commit/54cc2c492e3f6b22b3836899f2ddf83b1296f173))
|
||||
* fix chat_max_tokens calculation method to show true input_max_tokens ([#1241](https://github.com/microsoft/RD-Agent/issues/1241)) ([7d749b8](https://github.com/microsoft/RD-Agent/commit/7d749b819557f1abfca58189ae2abf2aec41fef5))
|
||||
* fix code diff bug ([#1115](https://github.com/microsoft/RD-Agent/issues/1115)) ([4603e88](https://github.com/microsoft/RD-Agent/commit/4603e88dbe910614f20a843f29463f17eebdda32))
|
||||
* fix mcts ([#1270](https://github.com/microsoft/RD-Agent/issues/1270)) ([c73f67a](https://github.com/microsoft/RD-Agent/commit/c73f67affee035def37474c66ebdd00dbc16c4ca))
|
||||
* fix some bugs in RD-Agent(Q) ([#1143](https://github.com/microsoft/RD-Agent/issues/1143)) ([44fd2ee](https://github.com/microsoft/RD-Agent/commit/44fd2ee68031599e106cbd99b8e86a110d8f2423))
|
||||
* **graph:** using assignment expression to avoid repeated function call ([#1174](https://github.com/microsoft/RD-Agent/issues/1174)) ([b4f57ce](https://github.com/microsoft/RD-Agent/commit/b4f57cec87bc61e8aa408319532cec055cb2d632))
|
||||
* handle mixed str and dict types in code_list ([#1279](https://github.com/microsoft/RD-Agent/issues/1279)) ([63ecb3b](https://github.com/microsoft/RD-Agent/commit/63ecb3bf26604d93f85595f6f6470c860be3c5ba))
|
||||
* handle None output and conditional step dump in LoopBase execution ([#1212](https://github.com/microsoft/RD-Agent/issues/1212)) ([68b6985](https://github.com/microsoft/RD-Agent/commit/68b69851916ed5bca42aab859ba7a9938bec4eb7))
|
||||
* handle the no-update case of root node in uncommited_rec_status ([#1062](https://github.com/microsoft/RD-Agent/issues/1062)) ([ead8dce](https://github.com/microsoft/RD-Agent/commit/ead8dced0e5b157b6e1bded380f440ee0b8a86f7))
|
||||
* handle ValueError in stdout shrinking and refactor shrink logic ([#1228](https://github.com/microsoft/RD-Agent/issues/1228)) ([bc7a3b4](https://github.com/microsoft/RD-Agent/commit/bc7a3b43b7cef45f036d508f95231b5885ad65f7))
|
||||
* ignore case when checking metric name ([#1160](https://github.com/microsoft/RD-Agent/issues/1160)) ([fc0df6e](https://github.com/microsoft/RD-Agent/commit/fc0df6e9fc7d8a9e7a0b4d4cb879cffbbcb9162f))
|
||||
* ignore class types when filtering workflow steps ([#1085](https://github.com/microsoft/RD-Agent/issues/1085)) ([64e3ec8](https://github.com/microsoft/RD-Agent/commit/64e3ec8f9afb5611814f9b64d50e6dc0685df8b2))
|
||||
* ignore RuntimeError for shared workspace double recovery ([#1140](https://github.com/microsoft/RD-Agent/issues/1140)) ([8fc1e9b](https://github.com/microsoft/RD-Agent/commit/8fc1e9bf8f5242e56d7bacf53cf58f9abe94e356))
|
||||
* improve the logic of json_schema and refine the reasoning extraction logic for reasoning model ([#1044](https://github.com/microsoft/RD-Agent/issues/1044)) ([12060b1](https://github.com/microsoft/RD-Agent/commit/12060b197ca618ca8901f93cde6bc2b42d79e4e9))
|
||||
* increase retry count in hypothesis_gen decorator to 10 ([#1230](https://github.com/microsoft/RD-Agent/issues/1230)) ([c4b8baa](https://github.com/microsoft/RD-Agent/commit/c4b8baaa5829567833ea2328fe89941423bf4cf2))
|
||||
* increase time default not controlled by LLM ([#1196](https://github.com/microsoft/RD-Agent/issues/1196)) ([8c62561](https://github.com/microsoft/RD-Agent/commit/8c62561d1c6bd3c8b3d354951cd154b08d567ef2))
|
||||
* insert await asyncio.sleep(0) to yield control in loop ([#1186](https://github.com/microsoft/RD-Agent/issues/1186)) ([5705be0](https://github.com/microsoft/RD-Agent/commit/5705be0512b788337c6798aea0bdf52791dd8e73))
|
||||
* jinja problem of enumerate ([#1216](https://github.com/microsoft/RD-Agent/issues/1216)) ([af9068c](https://github.com/microsoft/RD-Agent/commit/af9068c0b5263c5f58a43ccd13c19808020f77aa))
|
||||
* kaggle competition metric direction ([#1195](https://github.com/microsoft/RD-Agent/issues/1195)) ([a933b6c](https://github.com/microsoft/RD-Agent/commit/a933b6cabe6f6b673a30601f9b0974bc3ca806ae))
|
||||
* merge candidates ([#1254](https://github.com/microsoft/RD-Agent/issues/1254)) ([5a78c89](https://github.com/microsoft/RD-Agent/commit/5a78c89cee1fb593e3503bd4266042ba1e29569a))
|
||||
* minor conflict in prompts ([#1081](https://github.com/microsoft/RD-Agent/issues/1081)) ([f821e4c](https://github.com/microsoft/RD-Agent/commit/f821e4c1c56462c54d5fbe15dd797c147334b182))
|
||||
* minor fix to runtime_environment ([#1089](https://github.com/microsoft/RD-Agent/issues/1089)) ([bff82ef](https://github.com/microsoft/RD-Agent/commit/bff82ef93e225c43c6b55bb642c484d5b88f3cff))
|
||||
* model/factor experiment filtering in Qlib proposals ([#1257](https://github.com/microsoft/RD-Agent/issues/1257)) ([0f722e1](https://github.com/microsoft/RD-Agent/commit/0f722e1ce713d2010fe8b8181b905145a1186f95))
|
||||
* move snapshot saving after step index update in loop execution ([#1206](https://github.com/microsoft/RD-Agent/issues/1206)) ([0e3a9af](https://github.com/microsoft/RD-Agent/commit/0e3a9afd0a30b5a12ef3431043405f3314b4c635))
|
||||
* move task cancellation to finally block and fix subprocess kill typo ([#1234](https://github.com/microsoft/RD-Agent/issues/1234)) ([fb628e3](https://github.com/microsoft/RD-Agent/commit/fb628e3bcaded1f292e5827f258fa7d5f9ed74a9))
|
||||
* package and timer bug ([#1092](https://github.com/microsoft/RD-Agent/issues/1092)) ([7faf6d9](https://github.com/microsoft/RD-Agent/commit/7faf6d9b215d678b8cb146270a3e917a62ac1d88))
|
||||
* path traversal risk ([#1050](https://github.com/microsoft/RD-Agent/issues/1050)) ([2f78216](https://github.com/microsoft/RD-Agent/commit/2f782169ebeb0453422621ac8ace06353ca72615))
|
||||
* prevent JSON content from being added multiple times during retries ([#1255](https://github.com/microsoft/RD-Agent/issues/1255)) ([9d46a68](https://github.com/microsoft/RD-Agent/commit/9d46a68a36f237ef99bbc4a78668d71339fa9f91))
|
||||
* prevent parallelism in feedback and record steps ([#1046](https://github.com/microsoft/RD-Agent/issues/1046)) ([d0272a9](https://github.com/microsoft/RD-Agent/commit/d0272a9de104a629ccd2652b9e95c9bb58ac6cb1))
|
||||
* prompt yaml ([#1112](https://github.com/microsoft/RD-Agent/issues/1112)) ([1f2c9b1](https://github.com/microsoft/RD-Agent/commit/1f2c9b17b8d5250dc2ff81ad564139746d11a7c3))
|
||||
* properly assign sota_exp_fb before None comparison ([#1037](https://github.com/microsoft/RD-Agent/issues/1037)) ([5d6a927](https://github.com/microsoft/RD-Agent/commit/5d6a927501e95b6afa520294d23fcf9ca16c69ae))
|
||||
* refine DSCoSTEER_eval prompts ([#1157](https://github.com/microsoft/RD-Agent/issues/1157)) ([c62e5fc](https://github.com/microsoft/RD-Agent/commit/c62e5fcc871d4f88babc5a4c9cf8e4655e8ba437))
|
||||
* refine prompt, equal lightgbm, discourage over hypertuning ([#1072](https://github.com/microsoft/RD-Agent/issues/1072)) ([56ba15a](https://github.com/microsoft/RD-Agent/commit/56ba15a03fc278e7d701b40bbb5209411b27e561))
|
||||
* refine prompt; runner focus on low hanging fruit ([#1076](https://github.com/microsoft/RD-Agent/issues/1076)) ([1778b8c](https://github.com/microsoft/RD-Agent/commit/1778b8c953888e9b3b91d28483e0b64d126e3eb6))
|
||||
* refine prompts and add additional package info ([#1179](https://github.com/microsoft/RD-Agent/issues/1179)) ([22428a4](https://github.com/microsoft/RD-Agent/commit/22428a45053b6eefcfb805802b8bef4384a1ddda))
|
||||
* refine task scheduling logic in MultiProcessEvolvingStrategy for… ([#1275](https://github.com/microsoft/RD-Agent/issues/1275)) ([417766e](https://github.com/microsoft/RD-Agent/commit/417766ee366d1fdf4a54e297a93d05cb606d5144))
|
||||
* refine the prompt to force complete code & refine the logic of running ([#1069](https://github.com/microsoft/RD-Agent/issues/1069)) ([1e61de3](https://github.com/microsoft/RD-Agent/commit/1e61de3e60566029f1c89ca2c747bfbf3a354693))
|
||||
* remove refine decision & bug fix ([#1031](https://github.com/microsoft/RD-Agent/issues/1031)) ([0059a6a](https://github.com/microsoft/RD-Agent/commit/0059a6aeb658a76bdc28cd7741a2bc9e6569363f))
|
||||
* remove unused imports in data science scenario module ([#1136](https://github.com/microsoft/RD-Agent/issues/1136)) ([2307237](https://github.com/microsoft/RD-Agent/commit/23072377659da0bd206dc64dd858c9da75283f39))
|
||||
* replace hardcoded ChromeDriver path with webdriver-manager ([#1271](https://github.com/microsoft/RD-Agent/issues/1271)) ([40876e2](https://github.com/microsoft/RD-Agent/commit/40876e2085fb0e30e46b69fec34208d7e0dd1162))
|
||||
* revert 2 commits ([#1239](https://github.com/microsoft/RD-Agent/issues/1239)) ([1265ae9](https://github.com/microsoft/RD-Agent/commit/1265ae94e357190132fb2cd9ba3579d353ed6cee))
|
||||
* revert to v10 setting ([#1220](https://github.com/microsoft/RD-Agent/issues/1220)) ([d868188](https://github.com/microsoft/RD-Agent/commit/d868188f9a6fd451d1daf1b1cc14017a50232b0d))
|
||||
* scheduler next selection parallel disorder ([#1028](https://github.com/microsoft/RD-Agent/issues/1028)) ([f468595](https://github.com/microsoft/RD-Agent/commit/f468595169512b89f436396ee976404879e00d7a))
|
||||
* set requires_documentation_search to None to disable feature in eval ([#1245](https://github.com/microsoft/RD-Agent/issues/1245)) ([e117234](https://github.com/microsoft/RD-Agent/commit/e1172343e483638dc24715402048ec7116e8a429))
|
||||
* skip res_ratio check if timer or res_time is None ([#1189](https://github.com/microsoft/RD-Agent/issues/1189)) ([17400a3](https://github.com/microsoft/RD-Agent/commit/17400a3dc46ab987ef4670cf697a22c7145858be))
|
||||
* split then sample & remove simple model guide in ds proposal ([#1034](https://github.com/microsoft/RD-Agent/issues/1034)) ([2dde8b8](https://github.com/microsoft/RD-Agent/commit/2dde8b84a1d08cf0ca39b2f50de64d053fd73ba8))
|
||||
* stop evolve if global timer is timeout ([#1039](https://github.com/microsoft/RD-Agent/issues/1039)) ([ad37417](https://github.com/microsoft/RD-Agent/commit/ad374176a14be1fa5aac43fd8df48f89b2a81fe0))
|
||||
* summary page bug ([#1219](https://github.com/microsoft/RD-Agent/issues/1219)) ([36fec9a](https://github.com/microsoft/RD-Agent/commit/36fec9afa6d740a9f1ac32ac661cf7ec9fdaefc8))
|
||||
* TypeError: cannot unpack non-iterable bool object ([#1036](https://github.com/microsoft/RD-Agent/issues/1036)) ([f4370a4](https://github.com/microsoft/RD-Agent/commit/f4370a4265c84cefc4844d21b7f296929ca7638c))
|
||||
* ui bug ([#1192](https://github.com/microsoft/RD-Agent/issues/1192)) ([ad901aa](https://github.com/microsoft/RD-Agent/commit/ad901aaf4f7b344b8171b98ea753fde67b058a9b))
|
||||
* update fallback criterion ([#1210](https://github.com/microsoft/RD-Agent/issues/1210)) ([05fca1a](https://github.com/microsoft/RD-Agent/commit/05fca1acced3d3cfddbab3871d3dcee597b675bd))
|
||||
* update requirements.txt's streamlit ([#1133](https://github.com/microsoft/RD-Agent/issues/1133)) ([512d08f](https://github.com/microsoft/RD-Agent/commit/512d08f56c210edfa2ff45c71e53724909f10d8f))
|
||||
* use CoSTEERSettings for DSRunnerCoSTEERSettings ([#1096](https://github.com/microsoft/RD-Agent/issues/1096)) ([152a70f](https://github.com/microsoft/RD-Agent/commit/152a70f25a090e175e7b55c2285ca710954be9cc))
|
||||
|
||||
## [0.7.0](https://github.com/microsoft/RD-Agent/compare/v0.6.1...v0.7.0) (2025-07-08)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* add code change summary ([#1000](https://github.com/microsoft/RD-Agent/issues/1000)) ([937ec26](https://github.com/microsoft/RD-Agent/commit/937ec263b215928633822c4d76ad4e47442c8198))
|
||||
* add hide_base_name option and update data folder prompts ([#1004](https://github.com/microsoft/RD-Agent/issues/1004)) ([2f61fa8](https://github.com/microsoft/RD-Agent/commit/2f61fa8cd90c91ad29f320ce9ea6c49f49ac9111))
|
||||
* added running time statistics for the DS scenario experiment ([#1007](https://github.com/microsoft/RD-Agent/issues/1007)) ([030abd8](https://github.com/microsoft/RD-Agent/commit/030abd87191377641a678c80852f5ecad84e7a6e))
|
||||
* merge code summary and support more traces ([#1025](https://github.com/microsoft/RD-Agent/issues/1025)) ([48201e7](https://github.com/microsoft/RD-Agent/commit/48201e79b55ff5a98dad51702a7d0ac6b1ddc9eb))
|
||||
* show first evo round codes diff ([#1009](https://github.com/microsoft/RD-Agent/issues/1009)) ([4844622](https://github.com/microsoft/RD-Agent/commit/4844622e5fd28d7cbaabd9d7888f8204c60b76b3))
|
||||
* try coder on whole data ([#1017](https://github.com/microsoft/RD-Agent/issues/1017)) ([4973e05](https://github.com/microsoft/RD-Agent/commit/4973e0532248c6172eec3bb70dffda052af2d14f))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* fix a minor bug in DS eval ([#1012](https://github.com/microsoft/RD-Agent/issues/1012)) ([5a520e9](https://github.com/microsoft/RD-Agent/commit/5a520e9d44899d44fddc0f2e5571596223161b71))
|
||||
* fix some bugs in quant scen ([#1026](https://github.com/microsoft/RD-Agent/issues/1026)) ([7b34d41](https://github.com/microsoft/RD-Agent/commit/7b34d418642d1c0c2986db9ecf6a5d9bc22cc3da))
|
||||
* support experimental support for Deepseek models and update docs about configuration ([#1024](https://github.com/microsoft/RD-Agent/issues/1024)) ([35cfc19](https://github.com/microsoft/RD-Agent/commit/35cfc193f9b35d786aeb7585334427ad358c982f))
|
||||
|
||||
## [0.6.1](https://github.com/microsoft/RD-Agent/compare/v0.6.0...v0.6.1) (2025-06-28)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* fix mount ([#1001](https://github.com/microsoft/RD-Agent/issues/1001)) ([4ae2f13](https://github.com/microsoft/RD-Agent/commit/4ae2f1303dfcbaea53d459be7c8e85bf85ce5f4f))
|
||||
* handle the bug of wrong dag_parant index ([#996](https://github.com/microsoft/RD-Agent/issues/996)) ([bda12ff](https://github.com/microsoft/RD-Agent/commit/bda12ffecf9ae116e0d04eece0c6a1b61413d916))
|
||||
* improve log folder sorting and selection UX ([#993](https://github.com/microsoft/RD-Agent/issues/993)) ([b116807](https://github.com/microsoft/RD-Agent/commit/b11680777f116b6c40f9e535e0da10c186c95050))
|
||||
|
||||
## [0.6.0](https://github.com/microsoft/RD-Agent/compare/v0.5.0...v0.6.0) (2025-06-26)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* async mechanism for multi-trace ([#981](https://github.com/microsoft/RD-Agent/issues/981)) ([9e60c32](https://github.com/microsoft/RD-Agent/commit/9e60c32cf348481eb55617809c059c359d7603b8))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* add async to direct_exp_gen avoid infinite loop ([#992](https://github.com/microsoft/RD-Agent/issues/992)) ([78c203d](https://github.com/microsoft/RD-Agent/commit/78c203d8eefbba67fc120b35cb25e85b2200ac49))
|
||||
* docker container cleanup to prevent accumulation and system slowdown ([#975](https://github.com/microsoft/RD-Agent/issues/975)) ([05cf094](https://github.com/microsoft/RD-Agent/commit/05cf094913e48c903c8a4476d6c609d8bfa10681))
|
||||
* fix a bug and update the docs ([#978](https://github.com/microsoft/RD-Agent/issues/978)) ([d1ae9e1](https://github.com/microsoft/RD-Agent/commit/d1ae9e1dcc2ccd1ffe05cb1c6db3e905fa70425c))
|
||||
* merge datascience v3 and v2 ([#974](https://github.com/microsoft/RD-Agent/issues/974)) ([1ba7548](https://github.com/microsoft/RD-Agent/commit/1ba754853ce2010ce1cb0bbd217b67689fa1ebdf))
|
||||
* refine details ([#979](https://github.com/microsoft/RD-Agent/issues/979)) ([25caa3d](https://github.com/microsoft/RD-Agent/commit/25caa3d00c255286dce27915b9355987b87ed2e8))
|
||||
* refine prompt ([#987](https://github.com/microsoft/RD-Agent/issues/987)) ([76df96e](https://github.com/microsoft/RD-Agent/commit/76df96ee88212a8aee7f518b9cacf80591dc2939))
|
||||
|
||||
## [0.5.0](https://github.com/microsoft/RD-Agent/compare/v0.4.0...v0.5.0) (2025-06-18)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* add a check for whether values in score_df are NaN ([#756](https://github.com/microsoft/RD-Agent/issues/756)) ([d9cc780](https://github.com/microsoft/RD-Agent/commit/d9cc78098beb27f3a1bf2f2d461302db177b7d41))
|
||||
* add competition level filter and extract constants to utils ([#869](https://github.com/microsoft/RD-Agent/issues/869)) ([b40b605](https://github.com/microsoft/RD-Agent/commit/b40b6055368e6c72d8435352104b1c281b06da7f))
|
||||
* add DocDev for auto-generating workspace documentation ([#781](https://github.com/microsoft/RD-Agent/issues/781)) ([bcba6ea](https://github.com/microsoft/RD-Agent/commit/bcba6eac32684ebb267c93b4e85dbfa9561d15d1))
|
||||
* add drafting pipeline ([#832](https://github.com/microsoft/RD-Agent/issues/832)) ([efedddf](https://github.com/microsoft/RD-Agent/commit/efedddf39bc19221fdffc2e39ee0a09097fc82b0))
|
||||
* add last_exp_fb to DSTrace and update feedback retrieval usage ([#910](https://github.com/microsoft/RD-Agent/issues/910)) ([10531fd](https://github.com/microsoft/RD-Agent/commit/10531fda9438c6915b26d5013bd2413e1333ceb9))
|
||||
* add mlflow logger in RD loop to log ([#815](https://github.com/microsoft/RD-Agent/issues/815)) ([b91b54f](https://github.com/microsoft/RD-Agent/commit/b91b54f355c26b751087d0c14774f466e82866de))
|
||||
* add naive experiment generator and update proposal configurations ([#759](https://github.com/microsoft/RD-Agent/issues/759)) ([75494f4](https://github.com/microsoft/RD-Agent/commit/75494f4fed5bc845acfd7f7bacef385f0f96c514))
|
||||
* add RD-Agent-Quant scenario ([#838](https://github.com/microsoft/RD-Agent/issues/838)) ([6e42d52](https://github.com/microsoft/RD-Agent/commit/6e42d523a85df67aa13927abbf0894564c71880e))
|
||||
* add reasoning_effort parameter to LiteLLMAPIBackend and LLMSett… ([#754](https://github.com/microsoft/RD-Agent/issues/754)) ([113889f](https://github.com/microsoft/RD-Agent/commit/113889fefe9b09aaea1b564704c81664b8f77ec5))
|
||||
* add reviewer in feedback ([#765](https://github.com/microsoft/RD-Agent/issues/765)) ([1a95bee](https://github.com/microsoft/RD-Agent/commit/1a95bee6aa6bc6f45fdeb484f3a6f81caa273038))
|
||||
* advanced checkpoint selectors ([#790](https://github.com/microsoft/RD-Agent/issues/790)) ([50ea033](https://github.com/microsoft/RD-Agent/commit/50ea0336e93d8cb39fb871e81a3f61abdf293bc7))
|
||||
* archive python and csv files in workspace to maintain results ([#814](https://github.com/microsoft/RD-Agent/issues/814)) ([67d0e01](https://github.com/microsoft/RD-Agent/commit/67d0e01e7c9237da1371d93cbf9d86f5f46faac4))
|
||||
* checkpoint selection ([#744](https://github.com/microsoft/RD-Agent/issues/744)) ([a15a06a](https://github.com/microsoft/RD-Agent/commit/a15a06ad643977db59d7cac9da52e637cf80395a))
|
||||
* custom data ([#810](https://github.com/microsoft/RD-Agent/issues/810)) ([6322916](https://github.com/microsoft/RD-Agent/commit/632291608cf605bd8bcfcab0017824823bdecdb8))
|
||||
* dump model ([#776](https://github.com/microsoft/RD-Agent/issues/776)) ([b49481e](https://github.com/microsoft/RD-Agent/commit/b49481e073e6f536d2b1b3bd2d01229ed05abdea))
|
||||
* enable to set different version of idea-proposal for multi traces ([#895](https://github.com/microsoft/RD-Agent/issues/895)) ([236c28f](https://github.com/microsoft/RD-Agent/commit/236c28f29c6bc5da62129632e464bbc32056ebdb))
|
||||
* enhance compatibility with more LLM models ([#905](https://github.com/microsoft/RD-Agent/issues/905)) ([8800624](https://github.com/microsoft/RD-Agent/commit/8800624ad4749d6e798785a082c9f94c306792ef))
|
||||
* idea pool integrated to exp_gen & add timer to RD-Agent & pause-resume to RD-loops ([#795](https://github.com/microsoft/RD-Agent/issues/795)) ([e62aefa](https://github.com/microsoft/RD-Agent/commit/e62aefa56e34ff45a8ed033f7bf28b95c8e63656))
|
||||
* joblib cache ([#749](https://github.com/microsoft/RD-Agent/issues/749)) ([83a0411](https://github.com/microsoft/RD-Agent/commit/83a041148ff908871b1906f9e6889d80ab513412))
|
||||
* log api status to mlflow ([#860](https://github.com/microsoft/RD-Agent/issues/860)) ([049921b](https://github.com/microsoft/RD-Agent/commit/049921beb0b4ed0ba1ab7508d9857d2c1e729349))
|
||||
* log reaching max time limit before breaking CoSTEER evolution ([#921](https://github.com/microsoft/RD-Agent/issues/921)) ([837fff2](https://github.com/microsoft/RD-Agent/commit/837fff29096fefe1369d386ef8a860395b737173))
|
||||
* merge failed and successful traces together ([#766](https://github.com/microsoft/RD-Agent/issues/766)) ([3a2aa8c](https://github.com/microsoft/RD-Agent/commit/3a2aa8cf0102647950b2dfc0007c118b0c799cd4))
|
||||
* merge selectively ([#888](https://github.com/microsoft/RD-Agent/issues/888)) ([06ba314](https://github.com/microsoft/RD-Agent/commit/06ba314ff0f91e7e78e8d456c719ac3194a8c774))
|
||||
* multi-trace online merge ([#886](https://github.com/microsoft/RD-Agent/issues/886)) ([2112d67](https://github.com/microsoft/RD-Agent/commit/2112d676d0938de6fea163b2e5eb9c36771e7041))
|
||||
* new proposal (structured outputs) prompts ([#887](https://github.com/microsoft/RD-Agent/issues/887)) ([150796a](https://github.com/microsoft/RD-Agent/commit/150796aaa72eaa5037fd7db8e785058fbc4d4967))
|
||||
* parallel loop running based on asyncio ([#932](https://github.com/microsoft/RD-Agent/issues/932)) ([c63e207](https://github.com/microsoft/RD-Agent/commit/c63e2071f3179feef69f88061c0172cb5c3157f2))
|
||||
* propose hypothesis across multiple parts in pipeline ([#827](https://github.com/microsoft/RD-Agent/issues/827)) ([acb0e21](https://github.com/microsoft/RD-Agent/commit/acb0e21a331410d044849e12e2887f41e5ff1c3a))
|
||||
* pull image with progress ([#777](https://github.com/microsoft/RD-Agent/issues/777)) ([5cad086](https://github.com/microsoft/RD-Agent/commit/5cad0860204ede974533dc7bdc9808cfd135fa24))
|
||||
* raise error when timeout in api call ([#793](https://github.com/microsoft/RD-Agent/issues/793)) ([eafd4df](https://github.com/microsoft/RD-Agent/commit/eafd4dfc6263f19a8cdaf27498a1d07b43815306))
|
||||
* raise policy violation ([#894](https://github.com/microsoft/RD-Agent/issues/894)) ([5b9d007](https://github.com/microsoft/RD-Agent/commit/5b9d0072aebe15369e9a0010af83e71684baeae7))
|
||||
* reanalyze competition info & pipeline coding evaluator prompt ([#837](https://github.com/microsoft/RD-Agent/issues/837)) ([f7b5258](https://github.com/microsoft/RD-Agent/commit/f7b52580080c75d311355bcc6193b49495801809))
|
||||
* refine merge ([#842](https://github.com/microsoft/RD-Agent/issues/842)) ([99463b4](https://github.com/microsoft/RD-Agent/commit/99463b46819b3a0dcb2bb12a823a9cdf7ec560b4))
|
||||
* refine prompt ([#760](https://github.com/microsoft/RD-Agent/issues/760)) ([a91b182](https://github.com/microsoft/RD-Agent/commit/a91b182c4c9510eb34e4aab956588e909fa5d70b))
|
||||
* replace hard-coded cache paths with dynamic cache_path config ([#952](https://github.com/microsoft/RD-Agent/issues/952)) ([db56894](https://github.com/microsoft/RD-Agent/commit/db568947f1084a80d603718f5a13fdbd72b90a47))
|
||||
* revert draft stage into a soft decay in hypothesis selection ([#849](https://github.com/microsoft/RD-Agent/issues/849)) ([d41db0c](https://github.com/microsoft/RD-Agent/commit/d41db0ca357b07091825ebd9d18c303b6db3cc6a))
|
||||
* trace merging ([#836](https://github.com/microsoft/RD-Agent/issues/836)) ([a3d5473](https://github.com/microsoft/RD-Agent/commit/a3d547369e408a05cff570c1239b6320be40418d))
|
||||
* truncate by time ([#863](https://github.com/microsoft/RD-Agent/issues/863)) ([2b9427a](https://github.com/microsoft/RD-Agent/commit/2b9427ae036ffe1e28a717502f45500fe91fe5ac))
|
||||
* update prompt to improve json respond format of some LLM models ([#928](https://github.com/microsoft/RD-Agent/issues/928)) ([0b84709](https://github.com/microsoft/RD-Agent/commit/0b84709e59c7abb9754961cd17cc9673fcf508aa))
|
||||
* using different chat model in different part ([#822](https://github.com/microsoft/RD-Agent/issues/822)) ([c052ea6](https://github.com/microsoft/RD-Agent/commit/c052ea6d1f8948183a4a6ebc873ec01b57373cce))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* 'DSProposalV2ExpGen' object has no attribute 'COMPONENT_TASK_MAP… ([#950](https://github.com/microsoft/RD-Agent/issues/950)) ([e353895](https://github.com/microsoft/RD-Agent/commit/e353895251f231fee85abdcb1b22b022a577af77))
|
||||
* adapting UI to mock trace ([#841](https://github.com/microsoft/RD-Agent/issues/841)) ([8a5754c](https://github.com/microsoft/RD-Agent/commit/8a5754c9b9c9410d0943aeed777a93c13422e54a))
|
||||
* add missing semicolon after chmod in env shell command ([#955](https://github.com/microsoft/RD-Agent/issues/955)) ([1128eaa](https://github.com/microsoft/RD-Agent/commit/1128eaa89ec1dcab4a05ef50d64c7f7e6aae88a8))
|
||||
* add time to timer when api timeout bug ([#826](https://github.com/microsoft/RD-Agent/issues/826)) ([f45d6ae](https://github.com/microsoft/RD-Agent/commit/f45d6ae6595c1c39b389485b637a0ae53ffc8782))
|
||||
* add wait_retry to exp_gen v2 ([#783](https://github.com/microsoft/RD-Agent/issues/783)) ([b9fb7cf](https://github.com/microsoft/RD-Agent/commit/b9fb7cf4e3070062d91b5b67d0f10d6266b45142))
|
||||
* adjust ds_trace lookup and add stderr redirect to mlebench command ([#853](https://github.com/microsoft/RD-Agent/issues/853)) ([4e53108](https://github.com/microsoft/RD-Agent/commit/4e53108e020db719b39cba3a67e0c6dae3de19cf))
|
||||
* align competion_full_desc and scenario_all_desc, remove redundant info in problems proposal ([#808](https://github.com/microsoft/RD-Agent/issues/808)) ([76d8536](https://github.com/microsoft/RD-Agent/commit/76d8536d9ec53952383019306781d49cb3e9f75c))
|
||||
* bug fix in timer start ([#807](https://github.com/microsoft/RD-Agent/issues/807)) ([9af7161](https://github.com/microsoft/RD-Agent/commit/9af7161eb57bdd2e24b072335e9d185951c32472))
|
||||
* bug in problem identification ([#806](https://github.com/microsoft/RD-Agent/issues/806)) ([e1d5a29](https://github.com/microsoft/RD-Agent/commit/e1d5a2914046476f2f10d5884ed3c3ff956d65ff))
|
||||
* conda error information ([#941](https://github.com/microsoft/RD-Agent/issues/941)) ([fd39a94](https://github.com/microsoft/RD-Agent/commit/fd39a947763fb4a9be87b907c399bebe384df505))
|
||||
* default cost to NaN when calculation fails in LiteLLM backend ([#912](https://github.com/microsoft/RD-Agent/issues/912)) ([51a4048](https://github.com/microsoft/RD-Agent/commit/51a4048129cbfbc3b84bcf50fd8866fafb3e2da3))
|
||||
* ds trace ([#929](https://github.com/microsoft/RD-Agent/issues/929)) ([127e441](https://github.com/microsoft/RD-Agent/commit/127e441602e21a46d6313ff39133ab8ca841937e))
|
||||
* duplicate model names test in pipeline coder & runner ([#763](https://github.com/microsoft/RD-Agent/issues/763)) ([be3ee9d](https://github.com/microsoft/RD-Agent/commit/be3ee9da9882edda3c06ff7d1099d1bbda2203c3))
|
||||
* filter system metadata dirs and init missing DSTrace attribute ([#946](https://github.com/microsoft/RD-Agent/issues/946)) ([10050ef](https://github.com/microsoft/RD-Agent/commit/10050ef368ae7ec07cbf20ac4e52e21c2875eaab))
|
||||
* fix a bug in docker result extraction ([#824](https://github.com/microsoft/RD-Agent/issues/824)) ([e1c0f98](https://github.com/microsoft/RD-Agent/commit/e1c0f9826abcbc11dda215a600a2637c9ac6e984))
|
||||
* fix competition metric direction ([#784](https://github.com/microsoft/RD-Agent/issues/784)) ([3be0057](https://github.com/microsoft/RD-Agent/commit/3be0057556f46c899065ee1c7f9bafe33e79249c))
|
||||
* fix model input shape bug and costeer_model bug ([#821](https://github.com/microsoft/RD-Agent/issues/821)) ([b34bd89](https://github.com/microsoft/RD-Agent/commit/b34bd895d6d9c326aab85856a15be0cb72b2c4c8))
|
||||
* fix some minor bugs ([#758](https://github.com/microsoft/RD-Agent/issues/758)) ([963f96e](https://github.com/microsoft/RD-Agent/commit/963f96e5596bee04074135c2a0e31a8adc39ad8c))
|
||||
* fix some minor bugs in qlib scenario ([#817](https://github.com/microsoft/RD-Agent/issues/817)) ([79962a7](https://github.com/microsoft/RD-Agent/commit/79962a7ca40c77a3997a68da9ad1b5ab16728483))
|
||||
* fix the bug in the regular expression matching for stdout ([#890](https://github.com/microsoft/RD-Agent/issues/890)) ([ee57e37](https://github.com/microsoft/RD-Agent/commit/ee57e37a22af874b262c033d1606dbe7799706db))
|
||||
* fix the bug of Exceed-LLM-Context in online merge of multi-tarce ([#892](https://github.com/microsoft/RD-Agent/issues/892)) ([f760a3e](https://github.com/microsoft/RD-Agent/commit/f760a3eff7bd927a31e4958ed2f706312e83e3e3))
|
||||
* fix the problems weights bug ([#898](https://github.com/microsoft/RD-Agent/issues/898)) ([013d79f](https://github.com/microsoft/RD-Agent/commit/013d79f12060e908aeb57c3eb1bb56eea86df086))
|
||||
* fixed CI execution failures caused by document builds ([#857](https://github.com/microsoft/RD-Agent/issues/857)) ([5c116b2](https://github.com/microsoft/RD-Agent/commit/5c116b24ce727f6ed9ef39d5aa5b60442038c344))
|
||||
* get_metric_direction for aerial-cactus-identification ([#970](https://github.com/microsoft/RD-Agent/issues/970)) ([70dc62d](https://github.com/microsoft/RD-Agent/commit/70dc62de5fbd4272ecda1b6fcbcf898b3624a991))
|
||||
* import path of T ([#787](https://github.com/microsoft/RD-Agent/issues/787)) ([ac008a6](https://github.com/microsoft/RD-Agent/commit/ac008a61d03b4737ab3d994024e922839d8f3fe1))
|
||||
* improve eval alignment check (e.g. small-scale finetuning) ([#802](https://github.com/microsoft/RD-Agent/issues/802)) ([d391578](https://github.com/microsoft/RD-Agent/commit/d3915788082de640a4ce1eea6d2e607319b89c3e))
|
||||
* improve file tree and _walk symlink handling ([#877](https://github.com/microsoft/RD-Agent/issues/877)) ([516cb69](https://github.com/microsoft/RD-Agent/commit/516cb69357483ddd99f84b221a056d8491c34f9b))
|
||||
* log info ([#965](https://github.com/microsoft/RD-Agent/issues/965)) ([f1dbc21](https://github.com/microsoft/RD-Agent/commit/f1dbc2100498e22c8e5edbb2e4563c99c3d54775))
|
||||
* main bug ([#938](https://github.com/microsoft/RD-Agent/issues/938)) ([c6d34d6](https://github.com/microsoft/RD-Agent/commit/c6d34d67b8aedf5496bf6a875915ce657fc58448))
|
||||
* non-exist variable test_eval.py ([#847](https://github.com/microsoft/RD-Agent/issues/847)) ([4948c38](https://github.com/microsoft/RD-Agent/commit/4948c38560f4cf021d9354b201b22dfa5ccb9441))
|
||||
* refine feedback prompt ([#901](https://github.com/microsoft/RD-Agent/issues/901)) ([12bb2c4](https://github.com/microsoft/RD-Agent/commit/12bb2c4a1494b9aa29962905abb5e433a60eb716))
|
||||
* refine the time/memory constraints prompt in hypothesis proposal ([#856](https://github.com/microsoft/RD-Agent/issues/856)) ([51ce8ef](https://github.com/microsoft/RD-Agent/commit/51ce8ef84b4fe6590ce20599a56eee596f2f04e6))
|
||||
* Set PYTHONPATH in env.run_ret_code call in FBWorkspace class ([#755](https://github.com/microsoft/RD-Agent/issues/755)) ([68b5018](https://github.com/microsoft/RD-Agent/commit/68b501889caca754f27b57d9ab6f72184e93b15c))
|
||||
* task_gen for better understanding ([#752](https://github.com/microsoft/RD-Agent/issues/752)) ([6bfc1e5](https://github.com/microsoft/RD-Agent/commit/6bfc1e570449ee69ac110a4ced9a7cecbc0e6a73))
|
||||
* trace list but ([#852](https://github.com/microsoft/RD-Agent/issues/852)) ([32cdc57](https://github.com/microsoft/RD-Agent/commit/32cdc575bde103d71a358d4d99bd413076328ebd))
|
||||
* typo in workflow ([#861](https://github.com/microsoft/RD-Agent/issues/861)) ([0e54c9f](https://github.com/microsoft/RD-Agent/commit/0e54c9fe41d25a4cc45ab9e61bb2c2c01b854751))
|
||||
* update DS env setup with competition volume and timeout ([#878](https://github.com/microsoft/RD-Agent/issues/878)) ([816ada0](https://github.com/microsoft/RD-Agent/commit/816ada096afabe90578672b0e61b656802a30b62))
|
||||
* update feedback.py ([#772](https://github.com/microsoft/RD-Agent/issues/772)) ([133778c](https://github.com/microsoft/RD-Agent/commit/133778c67ee3349f1c2fe029bcf6a9ee14568efe))
|
||||
* update metric direction to return bool ([#791](https://github.com/microsoft/RD-Agent/issues/791)) ([0bf365e](https://github.com/microsoft/RD-Agent/commit/0bf365e7830aa86d2350b9d1c47410af46b3a7e8))
|
||||
* update runner max loop to 1 in DS scenario ([#820](https://github.com/microsoft/RD-Agent/issues/820)) ([3da378e](https://github.com/microsoft/RD-Agent/commit/3da378e986e8b776a17dbc694d29ef211192ed3e))
|
||||
* use fallback messages for missing submission and scores files ([#882](https://github.com/microsoft/RD-Agent/issues/882)) ([898fdea](https://github.com/microsoft/RD-Agent/commit/898fdeae80801d537ebc5c4a3b7df9de74c3403a))
|
||||
* use simple stdout and stderr ([#966](https://github.com/microsoft/RD-Agent/issues/966)) ([0b1c445](https://github.com/microsoft/RD-Agent/commit/0b1c445f1f0c212887ffff9f8fac44236df3607c))
|
||||
* use trace count as index ([#909](https://github.com/microsoft/RD-Agent/issues/909)) ([b87de56](https://github.com/microsoft/RD-Agent/commit/b87de56e54b206b3aada53850804474eff80b96d))
|
||||
* wrong variable test_eval.py ([#846](https://github.com/microsoft/RD-Agent/issues/846)) ([808ea6c](https://github.com/microsoft/RD-Agent/commit/808ea6cba541e60c35dd283cee9098ce46f2a59e))
|
||||
|
||||
## [0.4.0](https://github.com/microsoft/RD-Agent/compare/v0.3.0...v0.4.0) (2025-04-04)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* (Kaggle) add base template for competition: tabular-playground-series-may-2022 ([#481](https://github.com/microsoft/RD-Agent/issues/481)) ([f3405ca](https://github.com/microsoft/RD-Agent/commit/f3405ca732eb0ddca8e18ea72f69cbd86055c4ab))
|
||||
* a unified CoSTEER to fit more scenarios ([#491](https://github.com/microsoft/RD-Agent/issues/491)) ([cddbd02](https://github.com/microsoft/RD-Agent/commit/cddbd02e3ad3ccf6ad01443777319dc5c7eb08a7))
|
||||
* add a new competition ([#474](https://github.com/microsoft/RD-Agent/issues/474)) ([2fc0d77](https://github.com/microsoft/RD-Agent/commit/2fc0d77c485a31f647e21f4578e2e326f7032964))
|
||||
* add a tool to enable saving workspace files into a specific folder ([#728](https://github.com/microsoft/RD-Agent/issues/728)) ([bca864b](https://github.com/microsoft/RD-Agent/commit/bca864b7edeafe3f88405efb695ca8acad6252f8))
|
||||
* add baseline score stat ([#590](https://github.com/microsoft/RD-Agent/issues/590)) ([2948026](https://github.com/microsoft/RD-Agent/commit/2948026c390d067b643f8c8247c1447f1dc023e4))
|
||||
* add configurable volume mode for Docker volumes in env.py ([#537](https://github.com/microsoft/RD-Agent/issues/537)) ([642a022](https://github.com/microsoft/RD-Agent/commit/642a02239431411b91959f23e69b454997ca75d5))
|
||||
* add constraint labels for semantic search ([#680](https://github.com/microsoft/RD-Agent/issues/680)) ([0584cfc](https://github.com/microsoft/RD-Agent/commit/0584cfcd13ca1a62c85390ea2ee7574370748d31))
|
||||
* add cross validation to workflow ([#700](https://github.com/microsoft/RD-Agent/issues/700)) ([82e9b00](https://github.com/microsoft/RD-Agent/commit/82e9b00be62b01673353a7aaa3ab0e2e3ecaf3ca))
|
||||
* add describe_data_folder_v2 ([#738](https://github.com/microsoft/RD-Agent/issues/738)) ([bc8e846](https://github.com/microsoft/RD-Agent/commit/bc8e8460e0246321792ff3347b1b8905416ad075))
|
||||
* add do_truncate control for the load function ([#656](https://github.com/microsoft/RD-Agent/issues/656)) ([2b960a5](https://github.com/microsoft/RD-Agent/commit/2b960a58dfdeba69522a0f72ecf0975bb6ae87ee))
|
||||
* add do_truncate control for the load function ([#656](https://github.com/microsoft/RD-Agent/issues/656)) ([2b960a5](https://github.com/microsoft/RD-Agent/commit/2b960a58dfdeba69522a0f72ecf0975bb6ae87ee))
|
||||
* add eda to data science scenario ([#639](https://github.com/microsoft/RD-Agent/issues/639)) ([35aa479](https://github.com/microsoft/RD-Agent/commit/35aa479f00edf118d43ec228e0a84c155332957a))
|
||||
* add hypothesis guidelines and rule-based ranking ([#746](https://github.com/microsoft/RD-Agent/issues/746)) ([c077b82](https://github.com/microsoft/RD-Agent/commit/c077b8239cc72904c4bc450845ed2a11aa5445f0))
|
||||
* Add line length limit to shrink_text function and settings ([#715](https://github.com/microsoft/RD-Agent/issues/715)) ([75ed5e1](https://github.com/microsoft/RD-Agent/commit/75ed5e1c2ce1bf20bb55190c10a4134e04694d2b))
|
||||
* add loop_n parameter to the main loop ([#611](https://github.com/microsoft/RD-Agent/issues/611)) ([778c166](https://github.com/microsoft/RD-Agent/commit/778c166962250e3b9e7ad85de37f62297d370b45))
|
||||
* add max time config to costeer in data science ([#645](https://github.com/microsoft/RD-Agent/issues/645)) ([534686c](https://github.com/microsoft/RD-Agent/commit/534686c2ba7d9fa979c0762ad3177c36f6d7f4cb))
|
||||
* add mlebench submission validitor ([#545](https://github.com/microsoft/RD-Agent/issues/545)) ([712d94a](https://github.com/microsoft/RD-Agent/commit/712d94a7d6f22187fc3d18bd434e71ec6997aa9f))
|
||||
* add model removal and adjust some framework logic ([#681](https://github.com/microsoft/RD-Agent/issues/681)) ([1edf881](https://github.com/microsoft/RD-Agent/commit/1edf881c63512d351c0dd074d7a1c0965ff3119b))
|
||||
* add output_path to load function of LoopBase ([#628](https://github.com/microsoft/RD-Agent/issues/628)) ([dd33726](https://github.com/microsoft/RD-Agent/commit/dd33726ac5de75dc2030d193d457d59490b3361e))
|
||||
* add pipeline coder ([#742](https://github.com/microsoft/RD-Agent/issues/742)) ([759f295](https://github.com/microsoft/RD-Agent/commit/759f295dbf1224e177006e72d694e42dd6f372b6))
|
||||
* add rank into report (mle_summary) ([#665](https://github.com/microsoft/RD-Agent/issues/665)) ([13f7922](https://github.com/microsoft/RD-Agent/commit/13f7922aaae9e4143aac4ad08ec1c556c2faf04e))
|
||||
* add restart and fix unzip ([#538](https://github.com/microsoft/RD-Agent/issues/538)) ([ed2c7d1](https://github.com/microsoft/RD-Agent/commit/ed2c7d175f1f44ca06ad7a63b08da12f6c4df9ab))
|
||||
* add retry mechanism with wait_retry decorator and refactor diff generation ([#572](https://github.com/microsoft/RD-Agent/issues/572)) ([de1cd72](https://github.com/microsoft/RD-Agent/commit/de1cd72f068ebd1e1bd5bc2ad2b12ae484d54831))
|
||||
* add the shape of the CSV to the dataset description ([#561](https://github.com/microsoft/RD-Agent/issues/561)) ([a10c881](https://github.com/microsoft/RD-Agent/commit/a10c881bd86796e6167257ad26dd165f7e46d813))
|
||||
* add timeout settings and cleanup step in data science runner ([#539](https://github.com/microsoft/RD-Agent/issues/539)) ([295abd5](https://github.com/microsoft/RD-Agent/commit/295abd56f7b58055bd27b247dfed47eb85e9b0cd))
|
||||
* add type checker to api backend & align litellm and old backend ([#647](https://github.com/microsoft/RD-Agent/issues/647)) ([d38eae9](https://github.com/microsoft/RD-Agent/commit/d38eae986a0ba69d71288fa09fcc21e227551a02))
|
||||
* align mlebench data and evaluation & several fix on kaggle workflow ([#477](https://github.com/microsoft/RD-Agent/issues/477)) ([f6c522b](https://github.com/microsoft/RD-Agent/commit/f6c522b651db3c1f6af6815347589917f46e433a))
|
||||
* **backend:** integrate LiteLLM API Backend ([#564](https://github.com/microsoft/RD-Agent/issues/564)) ([f477687](https://github.com/microsoft/RD-Agent/commit/f4776879c76a213d53875b307c94be1ea5cfd9ba))
|
||||
* base data science scenario UI ([#525](https://github.com/microsoft/RD-Agent/issues/525)) ([39917b3](https://github.com/microsoft/RD-Agent/commit/39917b354b22a8488a17396fe2245cb41e3def03))
|
||||
* condaenv & full docker env ([#668](https://github.com/microsoft/RD-Agent/issues/668)) ([084dd6d](https://github.com/microsoft/RD-Agent/commit/084dd6d748a89492ea0888acb316b9bb9efeb62f))
|
||||
* diff mode fix ([#569](https://github.com/microsoft/RD-Agent/issues/569)) ([0c509f5](https://github.com/microsoft/RD-Agent/commit/0c509f599ce19303b44d8192ec3eb634c24992d6))
|
||||
* display LLM prompt ([#676](https://github.com/microsoft/RD-Agent/issues/676)) ([8c93bba](https://github.com/microsoft/RD-Agent/commit/8c93bba82e185edcf4204cc574df5f41bcdfa9d2))
|
||||
* Dynamically find and use sample submission file in eval tests ([#542](https://github.com/microsoft/RD-Agent/issues/542)) ([5f12b44](https://github.com/microsoft/RD-Agent/commit/5f12b44c89dd26b250e914192f9beb2da38fb3ab))
|
||||
* end-to-end optimization ([#473](https://github.com/microsoft/RD-Agent/issues/473)) ([d41343a](https://github.com/microsoft/RD-Agent/commit/d41343a63d87bf3479f5ec30745ea788580495bf))
|
||||
* Enhance eval script with file cleanup and detailed submission checks ([#529](https://github.com/microsoft/RD-Agent/issues/529)) ([cf2ff92](https://github.com/microsoft/RD-Agent/commit/cf2ff9213d3a8b0fad64df7cae0c35f996d72e27))
|
||||
* exclude invalid session log folder ([#554](https://github.com/microsoft/RD-Agent/issues/554)) ([fa86e4d](https://github.com/microsoft/RD-Agent/commit/fa86e4d1805000e0e5779c662ccbb5273fda623c))
|
||||
* improve the framework's ability to adaptively adjust the model ([#629](https://github.com/microsoft/RD-Agent/issues/629)) ([93806f3](https://github.com/microsoft/RD-Agent/commit/93806f33a1e0f29a125e29303d4b984a9817c3c0))
|
||||
* independent use_azure_token_provider on chat and embedding ([#452](https://github.com/microsoft/RD-Agent/issues/452)) ([d223004](https://github.com/microsoft/RD-Agent/commit/d223004917692e231b251330cbc8676081d5a10d))
|
||||
* integrate azure deepseek r1 ([#591](https://github.com/microsoft/RD-Agent/issues/591)) ([e79ce5c](https://github.com/microsoft/RD-Agent/commit/e79ce5c38539138abe04eb9809fbde437e97bbb7))
|
||||
* kaggle refactor ([#489](https://github.com/microsoft/RD-Agent/issues/489)) ([1b057d0](https://github.com/microsoft/RD-Agent/commit/1b057d0d63a861fba4b3cb59c6c5fc1a0e3da383))
|
||||
* **kaggle:** several update in kaggle scenarios ([#476](https://github.com/microsoft/RD-Agent/issues/476)) ([245d211](https://github.com/microsoft/RD-Agent/commit/245d211dcbfb18ebcc554247a0e3a8dbecf6f3bd))
|
||||
* loader prompt & simplify YAML loading and update data loader specifications ([#736](https://github.com/microsoft/RD-Agent/issues/736)) ([86f8bbf](https://github.com/microsoft/RD-Agent/commit/86f8bbf15895e7c198f9bc395d055ca5f02a5bb6))
|
||||
* make spec optional ([#719](https://github.com/microsoft/RD-Agent/issues/719)) ([a16b70f](https://github.com/microsoft/RD-Agent/commit/a16b70ff34c66d7e1c4c7ff5236eca8e7d8abea9))
|
||||
* Make system prompt role customizable in LLM settings ([#632](https://github.com/microsoft/RD-Agent/issues/632)) ([e4acd92](https://github.com/microsoft/RD-Agent/commit/e4acd92cc5eec6db5c29cb2d4788020fb89099b7))
|
||||
* multi log folder, replace "epxx" in workspace path ([#555](https://github.com/microsoft/RD-Agent/issues/555)) ([8a69c9c](https://github.com/microsoft/RD-Agent/commit/8a69c9c9630860c9b644356e1f71654aea222328))
|
||||
* new exp gen v2 implementation ([#725](https://github.com/microsoft/RD-Agent/issues/725)) ([5dcc2d5](https://github.com/microsoft/RD-Agent/commit/5dcc2d5fa63bbe9ae8c4817d9b40b77600440edb))
|
||||
* new-york-city-taxi-fare-prediction_template ([#488](https://github.com/microsoft/RD-Agent/issues/488)) ([a9caab7](https://github.com/microsoft/RD-Agent/commit/a9caab7bc5dc86f395a008e523355922137aef17))
|
||||
* out spec change for o1-preview ([#666](https://github.com/microsoft/RD-Agent/issues/666)) ([22894bd](https://github.com/microsoft/RD-Agent/commit/22894bdbee26b9cad73646d2975857787e515f75))
|
||||
* refactor for general data science ([#498](https://github.com/microsoft/RD-Agent/issues/498)) ([7002dc4](https://github.com/microsoft/RD-Agent/commit/7002dc4981a4f72096b438d2fe4fd9ff268c54f3))
|
||||
* refine logic for qlib_factor_from_report ([#463](https://github.com/microsoft/RD-Agent/issues/463)) ([21348d8](https://github.com/microsoft/RD-Agent/commit/21348d89e0e0eec1b4fab4e7a497f1eb34b8fe72))
|
||||
* run benchmark on gpt-4o & llama 3.1 ([#497](https://github.com/microsoft/RD-Agent/issues/497)) ([64af0b5](https://github.com/microsoft/RD-Agent/commit/64af0b5529b687cce8b5b7a1893946e15edca626))
|
||||
* summary and UI update ([#581](https://github.com/microsoft/RD-Agent/issues/581)) ([efa51f9](https://github.com/microsoft/RD-Agent/commit/efa51f9c259a06fe219f3137f0a1005e50d2bfdd))
|
||||
* template changes for some kaggle competitions ([#484](https://github.com/microsoft/RD-Agent/issues/484)) ([2e38000](https://github.com/microsoft/RD-Agent/commit/2e38000091030811fc081d72016c7bbadf7efd50))
|
||||
* track and log accumulated completion cost in LiteLLMAPIBackend ([#727](https://github.com/microsoft/RD-Agent/issues/727)) ([b294a95](https://github.com/microsoft/RD-Agent/commit/b294a95e0b7b2ef96af355cebac92d9c87f3acab))
|
||||
* update prompts and descriptions for data science components ([#731](https://github.com/microsoft/RD-Agent/issues/731)) ([c20e226](https://github.com/microsoft/RD-Agent/commit/c20e226c3e7771c9fcd1c879a8937e4694dc03eb))
|
||||
* variable printing tool of data_science coder testing ([#658](https://github.com/microsoft/RD-Agent/issues/658)) ([116c061](https://github.com/microsoft/RD-Agent/commit/116c06190b01f0b621c021726a1be23458ab1154))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* a default conf in scen qlib ([#503](https://github.com/microsoft/RD-Agent/issues/503)) ([d64a228](https://github.com/microsoft/RD-Agent/commit/d64a228525cbedd7687c1e06132eacd0d0647697))
|
||||
* a small bug in exp_gen ([#606](https://github.com/microsoft/RD-Agent/issues/606)) ([f734dde](https://github.com/microsoft/RD-Agent/commit/f734dde0b0101e13f38151468c8ddf9e23af26ac))
|
||||
* add check when retrying gen model codes ([#699](https://github.com/microsoft/RD-Agent/issues/699)) ([3b82f15](https://github.com/microsoft/RD-Agent/commit/3b82f159474087902d3c6007d370e3282b549015))
|
||||
* add DSExperiment type check and directory validation in log proc… ([#535](https://github.com/microsoft/RD-Agent/issues/535)) ([f59b12c](https://github.com/microsoft/RD-Agent/commit/f59b12c9cc9afde82b74bc133797ff1396678627))
|
||||
* add ensemble test, change to "use cross-validation if possible" in workflow spec ([#634](https://github.com/microsoft/RD-Agent/issues/634)) ([acc97a8](https://github.com/microsoft/RD-Agent/commit/acc97a8217253497afedcfa829902b4432e1031e))
|
||||
* add force parameter for cache_with_pickle & using cache when get kaggle leaderboard ([#687](https://github.com/microsoft/RD-Agent/issues/687)) ([c8841e5](https://github.com/microsoft/RD-Agent/commit/c8841e590a925200859acba9fda4a17d4c3aa1c7))
|
||||
* add metric name check for valid scores ([#724](https://github.com/microsoft/RD-Agent/issues/724)) ([acc2ffb](https://github.com/microsoft/RD-Agent/commit/acc2ffbde4df3b53654559d14cd035ee6be6b35e))
|
||||
* add retry mechanism for GPU device check in DockerEnv ([#573](https://github.com/microsoft/RD-Agent/issues/573)) ([a780cfb](https://github.com/microsoft/RD-Agent/commit/a780cfb621dc487cc17072bfd4aedd7d581249ab))
|
||||
* add scores.csv checking in ensemble_test ([#567](https://github.com/microsoft/RD-Agent/issues/567)) ([01808b4](https://github.com/microsoft/RD-Agent/commit/01808b47c314d1daffacc0a65e0ab934a1c41d65))
|
||||
* add stdout context length setting and improve text shrinking logic ([#559](https://github.com/microsoft/RD-Agent/issues/559)) ([4ac26a6](https://github.com/microsoft/RD-Agent/commit/4ac26a65c1f18f7513480dd562566c8a96298aa7))
|
||||
* align components' name ([#701](https://github.com/microsoft/RD-Agent/issues/701)) ([295a114](https://github.com/microsoft/RD-Agent/commit/295a1148c53d00b716b2d540573a7f43e7e2d762))
|
||||
* auto continue small bug ([#598](https://github.com/microsoft/RD-Agent/issues/598)) ([75eaecf](https://github.com/microsoft/RD-Agent/commit/75eaecf36b9f70dfc2d7fedd35836acdb05f89d6))
|
||||
* avoid try-except in ensemble eval prompts ([#637](https://github.com/microsoft/RD-Agent/issues/637)) ([5c58d6e](https://github.com/microsoft/RD-Agent/commit/5c58d6e524ef848024578033ab6d47bc9b220822))
|
||||
* avoid warning for missing llama installation when not in use ([#509](https://github.com/microsoft/RD-Agent/issues/509)) ([5ec3422](https://github.com/microsoft/RD-Agent/commit/5ec342224c2c8c4cf591f1eae673e25b14218726))
|
||||
* change devault to default ([#688](https://github.com/microsoft/RD-Agent/issues/688)) ([7f401cd](https://github.com/microsoft/RD-Agent/commit/7f401cd1c3b333285acf6d6e57654f4b9f0cb6c5))
|
||||
* change ensemble test ([#622](https://github.com/microsoft/RD-Agent/issues/622)) ([5de3595](https://github.com/microsoft/RD-Agent/commit/5de35953ed0d3e2e1f4dff0e0522f2d6475079ec))
|
||||
* change summary info of log folder ([#552](https://github.com/microsoft/RD-Agent/issues/552)) ([0eb258d](https://github.com/microsoft/RD-Agent/commit/0eb258d734e9a1280a238b9a6f63eb33047ee0a7))
|
||||
* clarify an ambiguous explanation ([#705](https://github.com/microsoft/RD-Agent/issues/705)) ([5dbfc68](https://github.com/microsoft/RD-Agent/commit/5dbfc6859cbf6cc31932dae30cf05506108fc871))
|
||||
* clarify cross_validation ([#644](https://github.com/microsoft/RD-Agent/issues/644)) ([906993e](https://github.com/microsoft/RD-Agent/commit/906993ef6482f88131d1af46f5bc66a77034b549))
|
||||
* coder prompt & model test text ([#583](https://github.com/microsoft/RD-Agent/issues/583)) ([0a41227](https://github.com/microsoft/RD-Agent/commit/0a41227f267050feaeeb47ddd4d749643eb9f198))
|
||||
* correct the configuration inheritance relationship ([#671](https://github.com/microsoft/RD-Agent/issues/671)) ([30b1ff8](https://github.com/microsoft/RD-Agent/commit/30b1ff8e1ce59b741e0b81481962063014641c0b))
|
||||
* default emb model ([#702](https://github.com/microsoft/RD-Agent/issues/702)) ([4329a72](https://github.com/microsoft/RD-Agent/commit/4329a722832a201b3fa6f9d8f9d8d46f78110410))
|
||||
* direct_exp_gen to json_target_type in DSExpGen class ([#661](https://github.com/microsoft/RD-Agent/issues/661)) ([428b74a](https://github.com/microsoft/RD-Agent/commit/428b74a988157ea864ebb40e828bd9f67589c863))
|
||||
* docker error will trigger retry and data science runner loop set to 3 ([#602](https://github.com/microsoft/RD-Agent/issues/602)) ([ad785e0](https://github.com/microsoft/RD-Agent/commit/ad785e03d5db05d9191d5e772e184532835a787b))
|
||||
* ensure expected type ([#593](https://github.com/microsoft/RD-Agent/issues/593)) ([098a9a6](https://github.com/microsoft/RD-Agent/commit/098a9a6618f70fa8dd276b9014b9e7ba9621553b))
|
||||
* filter empty log traces in ds UI ([#533](https://github.com/microsoft/RD-Agent/issues/533)) ([1a2057c](https://github.com/microsoft/RD-Agent/commit/1a2057c9fc11edc4637f0baaa6dd226eb049c36e))
|
||||
* fix a bug in cross validation ([#618](https://github.com/microsoft/RD-Agent/issues/618)) ([05a4f10](https://github.com/microsoft/RD-Agent/commit/05a4f101e0b64b860ad03294619b2350004657e8))
|
||||
* fix a bug in ensemble test script ([#713](https://github.com/microsoft/RD-Agent/issues/713)) ([ad32100](https://github.com/microsoft/RD-Agent/commit/ad321000acbd9291d22fe03a9c60e57c70511c73))
|
||||
* fix a bug in initial tasks ([#635](https://github.com/microsoft/RD-Agent/issues/635)) ([edb552e](https://github.com/microsoft/RD-Agent/commit/edb552ed283119444f357fbd0b6170b2ad97712a))
|
||||
* fix a bug in kaggle conf ([#459](https://github.com/microsoft/RD-Agent/issues/459)) ([b4ed32b](https://github.com/microsoft/RD-Agent/commit/b4ed32b17ef07d8557450063765585a48d5fcd32))
|
||||
* fix a bug in progress_bar filter ([#712](https://github.com/microsoft/RD-Agent/issues/712)) ([ba5a84d](https://github.com/microsoft/RD-Agent/commit/ba5a84dee59c39cc2a8c0d428a82da1f899ce537))
|
||||
* fix a bug in proposal (add last loop's exception to last task desc) ([#596](https://github.com/microsoft/RD-Agent/issues/596)) ([419186f](https://github.com/microsoft/RD-Agent/commit/419186ffb985fe5a0aa0f7fe59c7a223e355492e))
|
||||
* fix a bug in regular expression exception processing ([#734](https://github.com/microsoft/RD-Agent/issues/734)) ([67d3702](https://github.com/microsoft/RD-Agent/commit/67d37027bbcd7294a5890a350fe16fe78e0dfa77))
|
||||
* fix a bug in threshold score display ([#592](https://github.com/microsoft/RD-Agent/issues/592)) ([0b0a2dc](https://github.com/microsoft/RD-Agent/commit/0b0a2dc512a5560a66464ad49de25d362d0dc17e))
|
||||
* fix a bug related to model_name in ensemble ([#692](https://github.com/microsoft/RD-Agent/issues/692)) ([c6ce473](https://github.com/microsoft/RD-Agent/commit/c6ce4733f32578298abe0b60f9d82611b793cc09))
|
||||
* fix a minor bug ([#694](https://github.com/microsoft/RD-Agent/issues/694)) ([1405d8d](https://github.com/microsoft/RD-Agent/commit/1405d8dafd99ecde6f3ba9dd76133d8830d03b47))
|
||||
* fix an error in model_coder prompt ([#690](https://github.com/microsoft/RD-Agent/issues/690)) ([4528826](https://github.com/microsoft/RD-Agent/commit/452882674e915dbd9e3399c26c70ce5bb86d012c))
|
||||
* fix combined_factors_df.pkl not loading in docker ([#697](https://github.com/microsoft/RD-Agent/issues/697)) ([3984b99](https://github.com/microsoft/RD-Agent/commit/3984b995aa74318b40de7712e100d4de5cc95b11))
|
||||
* fix docs build error ([#711](https://github.com/microsoft/RD-Agent/issues/711)) ([c9e1d32](https://github.com/microsoft/RD-Agent/commit/c9e1d32d6b63560350cc7cb799c3a908e2c04e42))
|
||||
* fix ExtendedSettingsConfigDict does not work ([#660](https://github.com/microsoft/RD-Agent/issues/660)) ([3a877f3](https://github.com/microsoft/RD-Agent/commit/3a877f383b908da8d027560714030b201946bb76))
|
||||
* fix kaggle templates path error ([#747](https://github.com/microsoft/RD-Agent/issues/747)) ([3b3f504](https://github.com/microsoft/RD-Agent/commit/3b3f5041514baf741fe2d4613fa651fb5d9c002d))
|
||||
* fix KeyError direct_exp_gen ([#735](https://github.com/microsoft/RD-Agent/issues/735)) ([7200682](https://github.com/microsoft/RD-Agent/commit/7200682ac4e60d3910c29a4f7c4a37b3d24e4224))
|
||||
* fix some bugs (ensemble output, HPO, model tuning) ([#648](https://github.com/microsoft/RD-Agent/issues/648)) ([818ee29](https://github.com/microsoft/RD-Agent/commit/818ee29f8e5d4765b9801463b85b42ee9516ec33))
|
||||
* fix some bugs in the ensemble component ([#595](https://github.com/microsoft/RD-Agent/issues/595)) ([c0990ab](https://github.com/microsoft/RD-Agent/commit/c0990abb06c73ae062d9a50f50cdfd6d04aded22))
|
||||
* fix some bugs in workflow unit test ([#624](https://github.com/microsoft/RD-Agent/issues/624)) ([f845dcc](https://github.com/microsoft/RD-Agent/commit/f845dcc0ee1b059b8b32485ad46bb90c7ae0fa78))
|
||||
* fix some description errors in direct_exp_gen ([#698](https://github.com/microsoft/RD-Agent/issues/698)) ([dfaacb6](https://github.com/microsoft/RD-Agent/commit/dfaacb6d06e5d5f55e950d7177570d1efebf958f))
|
||||
* fix some minor bugs and add AutoML & cross-validation ([#604](https://github.com/microsoft/RD-Agent/issues/604)) ([18c5ef2](https://github.com/microsoft/RD-Agent/commit/18c5ef268d40efe7bb9ee18aa0d250732bdda6fa))
|
||||
* fix submission file search and add TODO in env.py ([#544](https://github.com/microsoft/RD-Agent/issues/544)) ([54d930e](https://github.com/microsoft/RD-Agent/commit/54d930e91e629f0fc2f8bdd0d0d62fcad1e99a9c))
|
||||
* fix task return dict with wrong format ([#558](https://github.com/microsoft/RD-Agent/issues/558)) ([2008244](https://github.com/microsoft/RD-Agent/commit/20082440a249dd0e5a7026c2d98c9de0288dd400))
|
||||
* fix the errors in the coder and evaluator of the five components ([#576](https://github.com/microsoft/RD-Agent/issues/576)) ([c487f83](https://github.com/microsoft/RD-Agent/commit/c487f835b651cdc40b95bbbe4efcb9a617be9e40))
|
||||
* handle division by zero in percentage calculations ([#550](https://github.com/microsoft/RD-Agent/issues/550)) ([de16c91](https://github.com/microsoft/RD-Agent/commit/de16c915e1716ef8cee43ce41069ea1a09cf1f24))
|
||||
* handle invalid regex patterns in filter_progress_bar function ([#579](https://github.com/microsoft/RD-Agent/issues/579)) ([b0daee0](https://github.com/microsoft/RD-Agent/commit/b0daee0d90e193ca1d028e01c31ebf368af89601))
|
||||
* Handle ValueError when resolving relative path for uri ([#585](https://github.com/microsoft/RD-Agent/issues/585)) ([4c7765a](https://github.com/microsoft/RD-Agent/commit/4c7765a12bda5dcfd9af72b292853d9bc28c5baf))
|
||||
* include data information in cache key generation ([#566](https://github.com/microsoft/RD-Agent/issues/566)) ([26dda46](https://github.com/microsoft/RD-Agent/commit/26dda4682b7b643c164589057cb568a4d9e55e17))
|
||||
* keep some txt files ([#557](https://github.com/microsoft/RD-Agent/issues/557)) ([54aba85](https://github.com/microsoft/RD-Agent/commit/54aba851c9fa194e318d37700307df59e06c6c84))
|
||||
* mle_score save problem ([#674](https://github.com/microsoft/RD-Agent/issues/674)) ([ca2e478](https://github.com/microsoft/RD-Agent/commit/ca2e478cf25c2c8511d5f027e32f8a98fc8e3a07))
|
||||
* move docker timeout message to __run() ([#620](https://github.com/microsoft/RD-Agent/issues/620)) ([585f4f9](https://github.com/microsoft/RD-Agent/commit/585f4f96e09f70d00eb397c10bf49c09973111df))
|
||||
* move mlebench check into runner ([#556](https://github.com/microsoft/RD-Agent/issues/556)) ([b0f7965](https://github.com/microsoft/RD-Agent/commit/b0f7965f650638273710302efee2e5da037368a2))
|
||||
* move next_component_required logic to DSTrace class and accurate implement ([#612](https://github.com/microsoft/RD-Agent/issues/612)) ([c20d311](https://github.com/microsoft/RD-Agent/commit/c20d311792f33b2ccccb466c6ec3155ff8be3213))
|
||||
* patching weird azure deployment ([#494](https://github.com/microsoft/RD-Agent/issues/494)) ([89c50ae](https://github.com/microsoft/RD-Agent/commit/89c50aee2ec8bfd1cb23767ddf7dcdd023daac8b))
|
||||
* qlib and other scenario bugs ([#636](https://github.com/microsoft/RD-Agent/issues/636)) ([98de31d](https://github.com/microsoft/RD-Agent/commit/98de31d4e577c8c450c9694f73a755c19af571f7))
|
||||
* refine prompt to generate the most simple task in init stage ([#546](https://github.com/microsoft/RD-Agent/issues/546)) ([9d6feed](https://github.com/microsoft/RD-Agent/commit/9d6feed28ce034db48482d8d9741ef8c72f4bddc))
|
||||
* replace API call with build_cls_from_json_with_retry function ([#548](https://github.com/microsoft/RD-Agent/issues/548)) ([eb72a47](https://github.com/microsoft/RD-Agent/commit/eb72a47fbf9c88dacea9691b8d7e92610492d190))
|
||||
* replace func "len()" in ensemble test code to support various data type ([#739](https://github.com/microsoft/RD-Agent/issues/739)) ([ab9c7b9](https://github.com/microsoft/RD-Agent/commit/ab9c7b955f78c5de7ec08a6c1a012a76badbdd0e))
|
||||
* return 1D embedding if create_embedding receive a string input ([#670](https://github.com/microsoft/RD-Agent/issues/670)) ([4a9c318](https://github.com/microsoft/RD-Agent/commit/4a9c3180ae4a4b043b1b4a89f51ee69cb6843142))
|
||||
* rich.print error when some control char in output ([#684](https://github.com/microsoft/RD-Agent/issues/684)) ([ec0cb2a](https://github.com/microsoft/RD-Agent/commit/ec0cb2a032824023dcd04a3acc93202471d1f90a))
|
||||
* Runnable on first complete & Rename method to next_incomplete_component for clarity ([#615](https://github.com/microsoft/RD-Agent/issues/615)) ([93d9f63](https://github.com/microsoft/RD-Agent/commit/93d9f63369a78f78e1a67ab548923bb994d1d3b4))
|
||||
* runner COSTEER evaluator ([#693](https://github.com/microsoft/RD-Agent/issues/693)) ([6a379ec](https://github.com/microsoft/RD-Agent/commit/6a379ec9b84d4e4944f1e412347aae4f5a93d476))
|
||||
* save only one mle_score pkl for a running exp ([#675](https://github.com/microsoft/RD-Agent/issues/675)) ([f87ab67](https://github.com/microsoft/RD-Agent/commit/f87ab676b73cce82bd9f997ac779e31c571b53c4))
|
||||
* Set default value for 'entry' parameter in Env.run method ([#643](https://github.com/microsoft/RD-Agent/issues/643)) ([e50d242](https://github.com/microsoft/RD-Agent/commit/e50d2424b849e4181d6ca02e9cace90236665924))
|
||||
* sort file name for cache reproduction ([#588](https://github.com/microsoft/RD-Agent/issues/588)) ([7158410](https://github.com/microsoft/RD-Agent/commit/7158410fbfdd84052f9a69cf1e04e09ac07ca598))
|
||||
* sota comparison logic ([#608](https://github.com/microsoft/RD-Agent/issues/608)) ([3575372](https://github.com/microsoft/RD-Agent/commit/35753722c0800d62855faeab996d513e62cfe7de))
|
||||
* target json type & round ([#662](https://github.com/microsoft/RD-Agent/issues/662)) ([58cb58f](https://github.com/microsoft/RD-Agent/commit/58cb58f966a1db26f5ea9662a54ba12bc921ee24))
|
||||
* templates bug ([#456](https://github.com/microsoft/RD-Agent/issues/456)) ([434a868](https://github.com/microsoft/RD-Agent/commit/434a8687eeda77e27b4938fb19694c15858ee446))
|
||||
* trace summary df showing in dsapp ([#551](https://github.com/microsoft/RD-Agent/issues/551)) ([177096d](https://github.com/microsoft/RD-Agent/commit/177096d55fecb8c7dab9650ef8f5a31024cd4c1c))
|
||||
* unzip kaggle data ([#464](https://github.com/microsoft/RD-Agent/issues/464)) ([3a9fc8e](https://github.com/microsoft/RD-Agent/commit/3a9fc8e73337d3757267b6f4482499499a1b6792))
|
||||
|
||||
## [0.3.0](https://github.com/microsoft/RD-Agent/compare/v0.2.1...v0.3.0) (2024-10-21)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* add a new template for kaggle ([#289](https://github.com/microsoft/RD-Agent/issues/289)) ([eee3ab5](https://github.com/microsoft/RD-Agent/commit/eee3ab5b25198224826cb7a8a17eab28bd5d1f7d))
|
||||
* add download submission.csv button for kaggle scenario ([#317](https://github.com/microsoft/RD-Agent/issues/317)) ([dcdcbe4](https://github.com/microsoft/RD-Agent/commit/dcdcbe46b4858bfb133ae3cca056e7f602d5cf63))
|
||||
* add kaggle command ([#271](https://github.com/microsoft/RD-Agent/issues/271)) ([0938394](https://github.com/microsoft/RD-Agent/commit/0938394b7084ffbf3294d8c23d2d34bf7322ca0b))
|
||||
* add kaggle tpl: feedback-prize ([#331](https://github.com/microsoft/RD-Agent/issues/331)) ([a288e39](https://github.com/microsoft/RD-Agent/commit/a288e399e6b0beec62729bd7d46b98a55de5ab79))
|
||||
* add more templates for kaggle ([#291](https://github.com/microsoft/RD-Agent/issues/291)) ([da752ec](https://github.com/microsoft/RD-Agent/commit/da752ec806e6f5f5679bc27ac1c072ed9a319251))
|
||||
* add normal rag into framework ([#360](https://github.com/microsoft/RD-Agent/issues/360)) ([91b0b1f](https://github.com/microsoft/RD-Agent/commit/91b0b1f66c3c1bf757cb64c4cfbdcaafe59eab74))
|
||||
* add qlib_factor_strategy ([#307](https://github.com/microsoft/RD-Agent/issues/307)) ([f8f59ff](https://github.com/microsoft/RD-Agent/commit/f8f59ff0a1be4428a68c8c27f220aabad0b6c9f0))
|
||||
* Add ranking in kaggle scenario ([#401](https://github.com/microsoft/RD-Agent/issues/401)) ([b16b4be](https://github.com/microsoft/RD-Agent/commit/b16b4beb402e0c27dfb39ee9d2a120f1b56d447c))
|
||||
* Add runtime measurement for each step and loop in RDLoop. ([#281](https://github.com/microsoft/RD-Agent/issues/281)) ([83058c8](https://github.com/microsoft/RD-Agent/commit/83058c864ceeec413dd29bf501030d5a7bd34679))
|
||||
* add s3e11 kaggle template ([#324](https://github.com/microsoft/RD-Agent/issues/324)) ([8c57524](https://github.com/microsoft/RD-Agent/commit/8c57524bead1c8f655a08763d608eb7a6dd5975e))
|
||||
* Added RepoAnalyzer to empower auto-summary of a workspace ([#264](https://github.com/microsoft/RD-Agent/issues/264)) ([0bd349a](https://github.com/microsoft/RD-Agent/commit/0bd349af50b9b881ba1774bdeb4d723529ef2aa9))
|
||||
* Added support for loading and storing RAG in Kaggle scenarios. ([#269](https://github.com/microsoft/RD-Agent/issues/269)) ([c4895de](https://github.com/microsoft/RD-Agent/commit/c4895de83f1ed000e563d42b3468a6bd9e5a4965))
|
||||
* announce Discord and WeChat ([#367](https://github.com/microsoft/RD-Agent/issues/367)) ([acac507](https://github.com/microsoft/RD-Agent/commit/acac5078a103b71afa6bd6c053b0766a6a7e609d))
|
||||
* auto submit result after one kaggle RDLoop ([#345](https://github.com/microsoft/RD-Agent/issues/345)) ([ab55d70](https://github.com/microsoft/RD-Agent/commit/ab55d7052b53a928b84dc5d5d0d2999d90ca9056))
|
||||
* better feedback & evaluation ([#346](https://github.com/microsoft/RD-Agent/issues/346)) ([cc9a8c1](https://github.com/microsoft/RD-Agent/commit/cc9a8c1eab3ca89f8c1e5de4a2bb4e7fcc0cc615))
|
||||
* Dynamic scenario based on task ([#392](https://github.com/microsoft/RD-Agent/issues/392)) ([665a037](https://github.com/microsoft/RD-Agent/commit/665a037e4fd7326c450e3fa0d0605eea26fd9ef3))
|
||||
* Factor Implement Search Enhancement ([#294](https://github.com/microsoft/RD-Agent/issues/294)) ([4ecf25f](https://github.com/microsoft/RD-Agent/commit/4ecf25f0acf2389a172b14d3dab20895daf2ab89))
|
||||
* Feature selection v3 to support all actions ([#280](https://github.com/microsoft/RD-Agent/issues/280)) ([0047641](https://github.com/microsoft/RD-Agent/commit/00476413fbf00e36e71ab3ccb48d4e766b6ccf4d))
|
||||
* fix some bugs and add original features' description ([#259](https://github.com/microsoft/RD-Agent/issues/259)) ([1a5f45a](https://github.com/microsoft/RD-Agent/commit/1a5f45a40d821c017bdba14af8c93710707c5ea5))
|
||||
* get kaggle notebooks & disscussion text for RAG ([#371](https://github.com/microsoft/RD-Agent/issues/371)) ([cead345](https://github.com/microsoft/RD-Agent/commit/cead3450a14bf4b142ac988c27fa098c7656a95c))
|
||||
* Iceberge competition ([#372](https://github.com/microsoft/RD-Agent/issues/372)) ([c10ea4f](https://github.com/microsoft/RD-Agent/commit/c10ea4f5d4cc56a75b47cf23c7084ee189ba1a25))
|
||||
* implement isolated model feature selection loop ([#370](https://github.com/microsoft/RD-Agent/issues/370)) ([cf1292d](https://github.com/microsoft/RD-Agent/commit/cf1292de1a0153ca14ea64971e73a1c93f7d89e3))
|
||||
* Initial version if Graph RAG in KAGGLE scenario ([#301](https://github.com/microsoft/RD-Agent/issues/301)) ([fd3c0fd](https://github.com/microsoft/RD-Agent/commit/fd3c0fd26eff7d3be72fa4f2a234e33b9f796627))
|
||||
* Integrate RAG into the Kaggle scenarios. ([#262](https://github.com/microsoft/RD-Agent/issues/262)) ([be0e48a](https://github.com/microsoft/RD-Agent/commit/be0e48a7dfbee2b5d2947d09115db5db2e5266f1))
|
||||
* Kaggle loop update (Feature & Model) ([#241](https://github.com/microsoft/RD-Agent/issues/241)) ([4cf22a6](https://github.com/microsoft/RD-Agent/commit/4cf22a65c964123b4267569ee02c0c7094c54ca4))
|
||||
* kaggle templates related ([#287](https://github.com/microsoft/RD-Agent/issues/287)) ([785fdc1](https://github.com/microsoft/RD-Agent/commit/785fdc144d16fa8454b7c9d2e53e78fe7f22a29a))
|
||||
* Model context for tuning and selection ([#284](https://github.com/microsoft/RD-Agent/issues/284)) ([f2831e7](https://github.com/microsoft/RD-Agent/commit/f2831e7442510668b0ca75953b3359894803ef3c))
|
||||
* Modify FactorRowCountEvaluator and FactorIndexEvaluator to return the ratio ([#328](https://github.com/microsoft/RD-Agent/issues/328)) ([8f43f8e](https://github.com/microsoft/RD-Agent/commit/8f43f8e87a92e05b541e925910608606ec8f6c4b))
|
||||
* New competition - Optiver ([#356](https://github.com/microsoft/RD-Agent/issues/356)) ([3705efe](https://github.com/microsoft/RD-Agent/commit/3705efe3b923748655a57d76b7a236e54d361831))
|
||||
* random forest for s3e11 ([#347](https://github.com/microsoft/RD-Agent/issues/347)) ([b57846d](https://github.com/microsoft/RD-Agent/commit/b57846d29314e9a5967945d1b4895f0f48c0f5ce))
|
||||
* refine the code in model description and fix some bugs in feedback.py ([#288](https://github.com/microsoft/RD-Agent/issues/288)) ([5b124d7](https://github.com/microsoft/RD-Agent/commit/5b124d7372137e4c613eb2749ddcc773922cc7b6))
|
||||
* refine the template in several Kaggle competitions ([#343](https://github.com/microsoft/RD-Agent/issues/343)) ([034f238](https://github.com/microsoft/RD-Agent/commit/034f238ed5ec351486b21250eabc75114961936c))
|
||||
* Revise to support better hypothesis proposal ([#390](https://github.com/microsoft/RD-Agent/issues/390)) ([c55ec0a](https://github.com/microsoft/RD-Agent/commit/c55ec0a0f577bbf7fc6228f7b87d2089ded83b31))
|
||||
* show workspace in demo ([#348](https://github.com/microsoft/RD-Agent/issues/348)) ([ddf567c](https://github.com/microsoft/RD-Agent/commit/ddf567c551b553788be022e9312c209ef6137d64))
|
||||
* support Multi output ([#330](https://github.com/microsoft/RD-Agent/issues/330)) ([3d36c45](https://github.com/microsoft/RD-Agent/commit/3d36c452ff0983800e5343834cc69f24a508ea70))
|
||||
* Supporting COVID-19 competition ([#374](https://github.com/microsoft/RD-Agent/issues/374)) ([a1b63db](https://github.com/microsoft/RD-Agent/commit/a1b63db79600edc9a74ba713c9d0be290214a592))
|
||||
* supporting Mnist competition ([#375](https://github.com/microsoft/RD-Agent/issues/375)) ([e958a34](https://github.com/microsoft/RD-Agent/commit/e958a34f5632a46ac43bff8e0d07d6ed020fdfc2))
|
||||
* Supporting Model Specifications ([#319](https://github.com/microsoft/RD-Agent/issues/319)) ([e126471](https://github.com/microsoft/RD-Agent/commit/e1264719e10b76158a91cd0ef331848e7c2de7c7))
|
||||
* supporting various Kaggle competitions & scenarios for RD-Agent ([#409](https://github.com/microsoft/RD-Agent/issues/409)) ([75eea22](https://github.com/microsoft/RD-Agent/commit/75eea22cc3d4e6f5a94c88cce915e27c507f8c50))
|
||||
* template for kaggle ([#308](https://github.com/microsoft/RD-Agent/issues/308)) ([ff97cf0](https://github.com/microsoft/RD-Agent/commit/ff97cf0155ab6941e4b5cf7d103575f934b70dc9))
|
||||
* use auto gen seed when using LLM cache ([#441](https://github.com/microsoft/RD-Agent/issues/441)) ([ca15365](https://github.com/microsoft/RD-Agent/commit/ca15365d23eeb094f42cf3dc8f5269b2f1c42bd3))
|
||||
* use unified pickle cacher & move llm config into a isolated config ([#424](https://github.com/microsoft/RD-Agent/issues/424)) ([2879ecf](https://github.com/microsoft/RD-Agent/commit/2879ecff816d97688b60909a79c7e568d42608a1))
|
||||
* xgboost gpu accelerate ([#359](https://github.com/microsoft/RD-Agent/issues/359)) ([56a5b8f](https://github.com/microsoft/RD-Agent/commit/56a5b8f9b2c6726cc64ec5b04b4ce7935d59b572))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* a bug of developer& edit s4e8 template ([#338](https://github.com/microsoft/RD-Agent/issues/338)) ([f12ce72](https://github.com/microsoft/RD-Agent/commit/f12ce726e7de96d478a232a3c27f92439820f8b4))
|
||||
* actively raised errors aer also considered as negative feedback. ([#268](https://github.com/microsoft/RD-Agent/issues/268)) ([46ec908](https://github.com/microsoft/RD-Agent/commit/46ec908e3594ac5e4cdc4057268e2f8800f5ed1f))
|
||||
* bug of saving preprocess cache files ([#310](https://github.com/microsoft/RD-Agent/issues/310)) ([5fb0608](https://github.com/microsoft/RD-Agent/commit/5fb0608f39f113cc9807fb1f381284a0bd4da318))
|
||||
* cache ([#383](https://github.com/microsoft/RD-Agent/issues/383)) ([f2a6e75](https://github.com/microsoft/RD-Agent/commit/f2a6e75b36ca96f7733b9c2a7154ac67bd2d7c6f))
|
||||
* change css tag of kaggle competition info crawler ([#306](https://github.com/microsoft/RD-Agent/issues/306)) ([1e3d38b](https://github.com/microsoft/RD-Agent/commit/1e3d38bf1ca3654f3a90ff392ecba1dbb4e80224))
|
||||
* debug dsagent ([#387](https://github.com/microsoft/RD-Agent/issues/387)) ([8fe9511](https://github.com/microsoft/RD-Agent/commit/8fe9511e606ba148c66f384add6ab94857079541))
|
||||
* eval_method cannot catch run factor error ([#260](https://github.com/microsoft/RD-Agent/issues/260)) ([2aaab31](https://github.com/microsoft/RD-Agent/commit/2aaab317ccb7a0121063bcd85fc36c21c7b8a391))
|
||||
* fix a bug in competition metric evaluation ([#407](https://github.com/microsoft/RD-Agent/issues/407)) ([94c47d6](https://github.com/microsoft/RD-Agent/commit/94c47d6fd5c3e38fc786a83e6d0d05e8d04498f3))
|
||||
* fix a bug in mini case ([#389](https://github.com/microsoft/RD-Agent/issues/389)) ([e75bb57](https://github.com/microsoft/RD-Agent/commit/e75bb5746f63933b750406bbd34ee63c5ba76b9f))
|
||||
* fix a bug in model tuning feedback ([#316](https://github.com/microsoft/RD-Agent/issues/316)) ([8aa088d](https://github.com/microsoft/RD-Agent/commit/8aa088da2dc7525a3970c01d01987246f47d6238))
|
||||
* fix a bug in scenario.py ([#388](https://github.com/microsoft/RD-Agent/issues/388)) ([999a1eb](https://github.com/microsoft/RD-Agent/commit/999a1eb0eff9088e1b02419db741db4acf8d9ff7))
|
||||
* fix a bug in the format of the model input ([#327](https://github.com/microsoft/RD-Agent/issues/327)) ([8f0574e](https://github.com/microsoft/RD-Agent/commit/8f0574eaaadb245b8c38e09ad4821306996d926f))
|
||||
* fix a small bug in cache using module name and function name as unique folder name ([#429](https://github.com/microsoft/RD-Agent/issues/429)) ([4f8134a](https://github.com/microsoft/RD-Agent/commit/4f8134a697d952f7ac824d7ebeec64bbc4545ab3))
|
||||
* fix a typo ([#362](https://github.com/microsoft/RD-Agent/issues/362)) ([9fafabd](https://github.com/microsoft/RD-Agent/commit/9fafabdf321b818bdd2211a2324d50cd0ebe1c1f))
|
||||
* fix cache result logic ([#430](https://github.com/microsoft/RD-Agent/issues/430)) ([5e34263](https://github.com/microsoft/RD-Agent/commit/5e342637dcc862679fd0642c6ba9ef048c984845))
|
||||
* fix command injection ([#421](https://github.com/microsoft/RD-Agent/issues/421)) ([52f30a6](https://github.com/microsoft/RD-Agent/commit/52f30a6184af1295be15e855a80b84bc424fc75d))
|
||||
* fix json load error ([#386](https://github.com/microsoft/RD-Agent/issues/386)) ([bba55fb](https://github.com/microsoft/RD-Agent/commit/bba55fb48fe105f4847c1b9c476eedc80835f523))
|
||||
* fix some bugs in feedback.py and refine the prompt ([#292](https://github.com/microsoft/RD-Agent/issues/292)) ([d834052](https://github.com/microsoft/RD-Agent/commit/d8340527f133dcc649d599d90d6402eddd37859e))
|
||||
* fix some bugs in knowledge base ([#378](https://github.com/microsoft/RD-Agent/issues/378)) ([fa6ff8e](https://github.com/microsoft/RD-Agent/commit/fa6ff8e591cf1847df77d73116649c5623161573))
|
||||
* fix some bugs in rag ([#399](https://github.com/microsoft/RD-Agent/issues/399)) ([194215c](https://github.com/microsoft/RD-Agent/commit/194215c4559aee5b6ece18d65c95fb30968e2db6))
|
||||
* fix some bugs in the entire loop ([#274](https://github.com/microsoft/RD-Agent/issues/274)) ([8a564ec](https://github.com/microsoft/RD-Agent/commit/8a564ece1d87b27ee98b76db317935e802468965))
|
||||
* fix some errors in scenario.py, proposal.py and runner.py and several complex competition scenarios([#365](https://github.com/microsoft/RD-Agent/issues/365)) ([2e383b1](https://github.com/microsoft/RD-Agent/commit/2e383b175d8448a67cb470f4e3ae8977d8ec6b5b))
|
||||
* improve_execution_time_in_kaggle_loop ([#279](https://github.com/microsoft/RD-Agent/issues/279)) ([4c8f998](https://github.com/microsoft/RD-Agent/commit/4c8f998c76f1e983a5687d2c65d3251750f2a9a0))
|
||||
* kaggle data mount problem ([#297](https://github.com/microsoft/RD-Agent/issues/297)) ([795df31](https://github.com/microsoft/RD-Agent/commit/795df311e3f93cd2f3fb51ba5698adaf10f6bd62))
|
||||
* Optiver fixes ([#357](https://github.com/microsoft/RD-Agent/issues/357)) ([b054017](https://github.com/microsoft/RD-Agent/commit/b054017463af0d1784407030f2477d212118f341))
|
||||
* partial bug in bench ([#368](https://github.com/microsoft/RD-Agent/issues/368)) ([af9808f](https://github.com/microsoft/RD-Agent/commit/af9808f98736a2df07e121c2f6d7bfeb7b7d3581))
|
||||
* preprocess output format & some mistake in spelling ([#358](https://github.com/microsoft/RD-Agent/issues/358)) ([b8b2cd6](https://github.com/microsoft/RD-Agent/commit/b8b2cd6ccd3b27aa73de847e50899a8a53b71b8f))
|
||||
* rag save file ([#385](https://github.com/microsoft/RD-Agent/issues/385)) ([1cb01dd](https://github.com/microsoft/RD-Agent/commit/1cb01dd6fe595f2f5fb86487601326611dd1a57a))
|
||||
* raise error in demo when no Metric in a Loop ([#313](https://github.com/microsoft/RD-Agent/issues/313)) ([e46a78e](https://github.com/microsoft/RD-Agent/commit/e46a78eb69271cb19978aab2f3b976c2870ca082))
|
||||
* refactor Bench ([#302](https://github.com/microsoft/RD-Agent/issues/302)) ([78a87f6](https://github.com/microsoft/RD-Agent/commit/78a87f624780ff67c0fa995ae4692678a120f99c))
|
||||
* refine some codes ([#353](https://github.com/microsoft/RD-Agent/issues/353)) ([866c2e6](https://github.com/microsoft/RD-Agent/commit/866c2e63ffa3876a3d16ad37f96da41d0558b714))
|
||||
* refine the prompt ([#286](https://github.com/microsoft/RD-Agent/issues/286)) ([77966c4](https://github.com/microsoft/RD-Agent/commit/77966c4f5e9f492c437c5b4b78d89c0f875ef0d8))
|
||||
* refine the ucb algorithm ([#406](https://github.com/microsoft/RD-Agent/issues/406)) ([14f7d97](https://github.com/microsoft/RD-Agent/commit/14f7d976e03c92d6e727524e0cdad8a03b585016))
|
||||
* revert model and make SOTA model available to COSTEER ([#351](https://github.com/microsoft/RD-Agent/issues/351)) ([3b7437b](https://github.com/microsoft/RD-Agent/commit/3b7437b87e685188259779cd85a78a0b592de9de))
|
||||
* stop using markup in docker env print ([#336](https://github.com/microsoft/RD-Agent/issues/336)) ([3009889](https://github.com/microsoft/RD-Agent/commit/3009889b5e2605b5427c76f3084e0e58026bb5ae))
|
||||
* support seed and fix absolute path ([#278](https://github.com/microsoft/RD-Agent/issues/278)) ([26352e1](https://github.com/microsoft/RD-Agent/commit/26352e13121cad5be95c0de78bb9f5dda4330614))
|
||||
* template for kaggle foreset & s4e9 ([#334](https://github.com/microsoft/RD-Agent/issues/334)) ([2393a41](https://github.com/microsoft/RD-Agent/commit/2393a41e7237615ced2c3fdd5c49308236b9f276))
|
||||
* test kaggle method ([#296](https://github.com/microsoft/RD-Agent/issues/296)) ([91a6196](https://github.com/microsoft/RD-Agent/commit/91a619618be1d7db660ea2b413a78dfaba9417a1))
|
||||
* update code to fix a small bug in model cache md5 hash ([#303](https://github.com/microsoft/RD-Agent/issues/303)) ([b00e4dc](https://github.com/microsoft/RD-Agent/commit/b00e4dc2eff5b16029a2a12a6589eadac5cfd148))
|
||||
* update new feature engineering code format ([#272](https://github.com/microsoft/RD-Agent/issues/272)) ([7850b80](https://github.com/microsoft/RD-Agent/commit/7850b8006a7c89d22629b345b4f361b0f35bc60d))
|
||||
* Update prompts.yaml to constrain only one model type ([#341](https://github.com/microsoft/RD-Agent/issues/341)) ([5b5dfee](https://github.com/microsoft/RD-Agent/commit/5b5dfeefbc7eb9dcbd9923544005c5d281262c03))
|
||||
* Update runner.py to fix a small bug ([#282](https://github.com/microsoft/RD-Agent/issues/282)) ([8aef3ab](https://github.com/microsoft/RD-Agent/commit/8aef3abcecd6002bd4bfeedcbe2c786d8bbfe2be))
|
||||
* Use fixed file name in model costeer & fixing cache ([#311](https://github.com/microsoft/RD-Agent/issues/311)) ([1f910a5](https://github.com/microsoft/RD-Agent/commit/1f910a5248bc576895ed66c2f7b2c3e046a2bc28))
|
||||
|
||||
|
||||
### Performance Improvements
|
||||
|
||||
* some small upgrade to factor costeer to improve the performance ([#420](https://github.com/microsoft/RD-Agent/issues/420)) ([9eb931f](https://github.com/microsoft/RD-Agent/commit/9eb931ffd971f252380dbd33ad1db259a4f229fd))
|
||||
|
||||
|
||||
### Reverts
|
||||
|
||||
* Revert feat: Factor Implement Search Enhancement ([#294](https://github.com/microsoft/RD-Agent/issues/294)) ([#305](https://github.com/microsoft/RD-Agent/issues/305)) ([f663cf4](https://github.com/microsoft/RD-Agent/commit/f663cf42a2f75cd52aef1c6b18be7c27f0641fed))
|
||||
|
||||
## [0.2.1](https://github.com/microsoft/RD-Agent/compare/v0.2.0...v0.2.1) (2024-09-10)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* default model value in config ([#256](https://github.com/microsoft/RD-Agent/issues/256)) ([c097585](https://github.com/microsoft/RD-Agent/commit/c097585f631f401c2c0966f6ad4c17286924f011))
|
||||
* fix_dotenv_error ([#257](https://github.com/microsoft/RD-Agent/issues/257)) ([923063c](https://github.com/microsoft/RD-Agent/commit/923063c1fd957c4ed42e97272c72b5e9545451dc))
|
||||
* readme ([#248](https://github.com/microsoft/RD-Agent/issues/248)) ([8cede22](https://github.com/microsoft/RD-Agent/commit/8cede2209922876490148459e1134da828e1fda0))
|
||||
|
||||
## [0.2.0](https://github.com/microsoft/RD-Agent/compare/v0.1.0...v0.2.0) (2024-09-07)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* add collect info ([#233](https://github.com/microsoft/RD-Agent/issues/233)) ([89f4af9](https://github.com/microsoft/RD-Agent/commit/89f4af90fb4d95a0689bf9efc8ffd9326469c0aa))
|
||||
* add cross validation for kaggle scenario ([#236](https://github.com/microsoft/RD-Agent/issues/236)) ([e0b03ba](https://github.com/microsoft/RD-Agent/commit/e0b03ba6b5c3d9aa552b99d470e106d4e348e64d))
|
||||
* add progress status for docker env ([#215](https://github.com/microsoft/RD-Agent/issues/215)) ([538d4ef](https://github.com/microsoft/RD-Agent/commit/538d4ef2e52de795b90d3f75b2e1e877ab85c18d))
|
||||
* Added loop code for Kaggle scene. ([#211](https://github.com/microsoft/RD-Agent/issues/211)) ([975c327](https://github.com/microsoft/RD-Agent/commit/975c32715e51aec6b49537401f5fc59115e04a01))
|
||||
* Demo display effect and usage ([#162](https://github.com/microsoft/RD-Agent/issues/162)) ([8cf122a](https://github.com/microsoft/RD-Agent/commit/8cf122a0155f434fa4477ae7a6d616b5caecd3e0))
|
||||
* piloting of the framework ([#227](https://github.com/microsoft/RD-Agent/issues/227)) ([e9b103e](https://github.com/microsoft/RD-Agent/commit/e9b103e684fdd2b98cd1a89971a3fce2d6e884a1))
|
||||
* support more models for kaggle scenario ([#223](https://github.com/microsoft/RD-Agent/issues/223)) ([e3a9659](https://github.com/microsoft/RD-Agent/commit/e3a96598c0720fe092ec86d7ca8c195c7d6bcc72))
|
||||
* update model_experiment.py to support basic EDA ([#220](https://github.com/microsoft/RD-Agent/issues/220)) ([bf2684c](https://github.com/microsoft/RD-Agent/commit/bf2684c4d55ab8e1048ac0291695475ad53b0cd6))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* fix some bugs in llm calling ([#217](https://github.com/microsoft/RD-Agent/issues/217)) ([7b010f8](https://github.com/microsoft/RD-Agent/commit/7b010f8b5940aba65a58f1d78192aa80bcd0e654))
|
||||
* package dependency. ([#234](https://github.com/microsoft/RD-Agent/issues/234)) ([46be295](https://github.com/microsoft/RD-Agent/commit/46be2952952af534fd8d98a656c704c688d7cbdd))
|
||||
* remove useless line ([#177](https://github.com/microsoft/RD-Agent/issues/177)) ([64e9a8e](https://github.com/microsoft/RD-Agent/commit/64e9a8e39a2072a962111db18f5b9565df5b0176))
|
||||
|
||||
## [0.1.0](https://github.com/microsoft/RD-Agent/compare/v0.0.1...v0.1.0) (2024-08-09)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* add entry for rdagent. ([#187](https://github.com/microsoft/RD-Agent/issues/187)) ([121b6d9](https://github.com/microsoft/RD-Agent/commit/121b6d98de38cd03be30cbee47b40baf39a2b60b))
|
||||
* change ui entry ([#197](https://github.com/microsoft/RD-Agent/issues/197)) ([fa5d335](https://github.com/microsoft/RD-Agent/commit/fa5d3354d22240888f4fc4007d9834f7424632aa))
|
||||
* remove pdfs and enable online pdf readings ([#183](https://github.com/microsoft/RD-Agent/issues/183)) ([18c0501](https://github.com/microsoft/RD-Agent/commit/18c05016a23d694c7b12759cf1322562dcffc56a))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* Fix a fail href in readme ([#189](https://github.com/microsoft/RD-Agent/issues/189)) ([1b89218](https://github.com/microsoft/RD-Agent/commit/1b89218f6bc697494f4a1b8a42ad18963002714f))
|
||||
* fix quick start problem ([#191](https://github.com/microsoft/RD-Agent/issues/191)) ([44f61bf](https://github.com/microsoft/RD-Agent/commit/44f61bfa1058a8efb59ca48b7f1417765aeea33e))
|
||||
* update command line in readme.md ([#192](https://github.com/microsoft/RD-Agent/issues/192)) ([9c45d24](https://github.com/microsoft/RD-Agent/commit/9c45d24a192da02f7d9765cb001097da1bc36c61))
|
||||
|
||||
## 0.0.1 (2024-08-08)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* Add description for scenario experiments. ([#174](https://github.com/microsoft/RD-Agent/issues/174)) ([fbd8c6d](https://github.com/microsoft/RD-Agent/commit/fbd8c6d87e1424c08997103b8e8fbf264858c4ed))
|
||||
* Added QlibFactorFromReportScenario and improved the report-factor loop. ([#161](https://github.com/microsoft/RD-Agent/issues/161)) ([882c79b](https://github.com/microsoft/RD-Agent/commit/882c79bf11583980e646b130f71cfa20201ffc7b))
|
||||
* filter feature which is high correlation to former implemented features ([#145](https://github.com/microsoft/RD-Agent/issues/145)) ([e818326](https://github.com/microsoft/RD-Agent/commit/e818326422740e04a4863f7c3c18744dde2ad98f))
|
||||
* Remove redundant 'key steps' section in frontend scene display. ([#169](https://github.com/microsoft/RD-Agent/issues/169)) ([e767005](https://github.com/microsoft/RD-Agent/commit/e76700513bee29232c93b97414419df330d9be8d))
|
||||
* streamlit webapp demo for different scenarios ([#135](https://github.com/microsoft/RD-Agent/issues/135)) ([d8da7db](https://github.com/microsoft/RD-Agent/commit/d8da7db865e6653fc4740efee9a843b69bd79699))
|
||||
* Uploaded Documentation, Updated Prompts & Some Code for model demo ([#144](https://github.com/microsoft/RD-Agent/issues/144)) ([529f935](https://github.com/microsoft/RD-Agent/commit/529f935aa98623f0dc1dda29eecee3ef738dd446))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* Add framework handling for task coding failure. ([#176](https://github.com/microsoft/RD-Agent/issues/176)) ([5e14fa5](https://github.com/microsoft/RD-Agent/commit/5e14fa54a9dd30a94aebe2643b8c9a3b85517a11))
|
||||
* Comprehensive update to factor extraction. ([#143](https://github.com/microsoft/RD-Agent/issues/143)) ([b5ea040](https://github.com/microsoft/RD-Agent/commit/b5ea04019fd5fa15c0f8b9a7e4f18f490f7057d4))
|
||||
* first round app folder cleaning ([#166](https://github.com/microsoft/RD-Agent/issues/166)) ([6a5a750](https://github.com/microsoft/RD-Agent/commit/6a5a75021912927deb5e8e4c7ad3ec4b51bfc788))
|
||||
* fix pickle problem ([#140](https://github.com/microsoft/RD-Agent/issues/140)) ([7ee4258](https://github.com/microsoft/RD-Agent/commit/7ee42587b60d94417f34332cee395cf210dc8a0e))
|
||||
* fix release CI ([#165](https://github.com/microsoft/RD-Agent/issues/165)) ([85d6a5e](https://github.com/microsoft/RD-Agent/commit/85d6a5ed91113fda34ae079b23c89aa24acd2cb2))
|
||||
* fix release CI error ([#160](https://github.com/microsoft/RD-Agent/issues/160)) ([1c9f8ef](https://github.com/microsoft/RD-Agent/commit/1c9f8ef287961731944acc9008496b4dddeddca7))
|
||||
* fix several bugs in data mining scenario ([#147](https://github.com/microsoft/RD-Agent/issues/147)) ([b233380](https://github.com/microsoft/RD-Agent/commit/b233380e2c66fb030db39424f0f040c86e37f5c4))
|
||||
* fix some small bugs in report-factor loop ([#152](https://github.com/microsoft/RD-Agent/issues/152)) ([a79f9f9](https://github.com/microsoft/RD-Agent/commit/a79f9f93406aff6305a76e6a6abd3852642e4c62))
|
||||
* fix_release_ci_error ([#150](https://github.com/microsoft/RD-Agent/issues/150)) ([4f82e99](https://github.com/microsoft/RD-Agent/commit/4f82e9960a2638af9d831581185ddd3bac5711fc))
|
||||
* Fixed some bugs introduced during refactoring. ([#167](https://github.com/microsoft/RD-Agent/issues/167)) ([f8f1445](https://github.com/microsoft/RD-Agent/commit/f8f1445283fb89aefeb2918243c35a219a51a56c))
|
||||
* optimize some prompts in factor loop. ([#158](https://github.com/microsoft/RD-Agent/issues/158)) ([c2c1330](https://github.com/microsoft/RD-Agent/commit/c2c13300b9ad315a663ec2d0eada414e56c6f54f))
|
||||
|
||||
|
||||
### Miscellaneous Chores
|
||||
|
||||
* release 0.0.1 ([1feacd3](https://github.com/microsoft/RD-Agent/commit/1feacd39b21193de11e9bbecf880ddf96d7c261c))
|
||||
@@ -0,0 +1,9 @@
|
||||
# Microsoft Open Source Code of Conduct
|
||||
|
||||
This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
|
||||
|
||||
Resources:
|
||||
|
||||
- [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/)
|
||||
- [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/)
|
||||
- Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns
|
||||
@@ -0,0 +1,50 @@
|
||||
# Contributing to RD-Agent
|
||||
|
||||
We welcome contributions and suggestions to improve RD-Agent. Whether it's solving an issue, addressing a bug, enhancing documentation, or even correcting a typo, every contribution is valuable and helps improve the project.
|
||||
|
||||
## Getting Started
|
||||
|
||||
To get started, you can explore the issues list or search for `TODO:` comments in the codebase by running the command:
|
||||
```sh
|
||||
grep -r "TODO:"
|
||||
```
|
||||
|
||||
## How to Contribute
|
||||
|
||||
1. **Fork the Repository**: Create a fork of the repository on GitHub.
|
||||
2. **Clone the Repository**: Clone your forked repository to your local machine.
|
||||
```sh
|
||||
git clone https://github.com/your-username/RD-Agent.git
|
||||
```
|
||||
3. **Create a Branch**: Create a new branch for your changes.
|
||||
```sh
|
||||
git checkout -b feature/your-feature-name
|
||||
```
|
||||
4. **Make Changes**: Make your changes to the codebase.
|
||||
5. **Commit Changes**: Commit your changes with a descriptive commit message.
|
||||
```sh
|
||||
git commit -m "Description of your changes"
|
||||
```
|
||||
6. **Push Changes**: Push your changes to your forked repository.
|
||||
```sh
|
||||
git push origin feature/your-feature-name
|
||||
```
|
||||
7. **Ensure CI Passes**: Make sure your code passes the automatic CI checks on GitHub.
|
||||
8. **Create a Pull Request**: Create a pull request from your forked repository to the main repository.
|
||||
|
||||
## Code of Conduct
|
||||
|
||||
Please adhere to the [Code of Conduct](CODE_OF_CONDUCT.md) in all your interactions with the project.
|
||||
|
||||
## Reporting Issues
|
||||
|
||||
If you encounter any issues or have suggestions for improvements, please open an issue on GitHub.
|
||||
|
||||
## Guidelines
|
||||
|
||||
- Ensure your code follows the project's coding standards.
|
||||
- Write clear and concise commit messages.
|
||||
- Update documentation as needed.
|
||||
- Test your changes thoroughly before submitting a pull request.
|
||||
|
||||
Thank you for contributing to RD-Agent!
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) Microsoft Corporation.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE
|
||||
@@ -0,0 +1,222 @@
|
||||
.PHONY: clean deepclean install init-qlib-env dev constraints black isort mypy ruff toml-sort lint pre-commit test-run test build upload docs-autobuild changelog docs-gen docs-mypy docs-coverage docs
|
||||
#You can modify it according to your terminal
|
||||
SHELL := /bin/bash
|
||||
|
||||
########################################################################################
|
||||
# Variables
|
||||
########################################################################################
|
||||
|
||||
# Determine whether to invoke pipenv based on CI environment variable and the availability of pipenv.
|
||||
PIPRUN := $(shell [ "$$CI" != "true" ] && command -v pipenv > /dev/null 2>&1 && echo "pipenv run")
|
||||
|
||||
# Get the Python version in `major.minor` format, using the environment variable or the virtual environment if exists.
|
||||
PYTHON_VERSION := $(shell echo $${PYTHON_VERSION:-$$(python -V 2>&1 | cut -d ' ' -f 2)} | cut -d '.' -f 1,2)
|
||||
|
||||
# Determine the constraints file based on the Python version.
|
||||
CONSTRAINTS_FILE := constraints/$(PYTHON_VERSION).txt
|
||||
|
||||
# Documentation target directory, will be adapted to specific folder for readthedocs.
|
||||
PUBLIC_DIR := $(shell [ "$$READTHEDOCS" = "True" ] && echo "$$READTHEDOCS_OUTPUT/html" || echo "public")
|
||||
|
||||
# URL and Path of changelog source code.
|
||||
CHANGELOG_URL := $(shell echo $${CI_PAGES_URL:-https://microsoft.github.io/rdagent}/_sources/changelog.md.txt)
|
||||
CHANGELOG_PATH := docs/changelog.md
|
||||
|
||||
########################################################################################
|
||||
# Development Environment Management
|
||||
########################################################################################
|
||||
|
||||
# Remove common intermediate files.
|
||||
clean:
|
||||
-rm -rf \
|
||||
$(PUBLIC_DIR) \
|
||||
.coverage \
|
||||
.mypy_cache \
|
||||
.pytest_cache \
|
||||
.ruff_cache \
|
||||
Pipfile* \
|
||||
coverage.xml \
|
||||
dist \
|
||||
release-notes.md
|
||||
find . -name '*.egg-info' -print0 | xargs -0 rm -rf
|
||||
find . -name '*.pyc' -print0 | xargs -0 rm -f
|
||||
find . -name '*.swp' -print0 | xargs -0 rm -f
|
||||
find . -name '.DS_Store' -print0 | xargs -0 rm -f
|
||||
find . -name '__pycache__' -print0 | xargs -0 rm -rf
|
||||
|
||||
# Remove pre-commit hook, virtual environment alongside itermediate files.
|
||||
deepclean: clean
|
||||
if command -v pre-commit > /dev/null 2>&1; then pre-commit uninstall --hook-type pre-push; fi
|
||||
if command -v pipenv >/dev/null 2>&1 && pipenv --venv >/dev/null 2>&1; then pipenv --rm; fi
|
||||
|
||||
# Install the package in editable mode.
|
||||
install:
|
||||
$(PIPRUN) pip install -e . -c $(CONSTRAINTS_FILE)
|
||||
|
||||
# Install the package in editable mode with specific optional dependencies.
|
||||
dev-%:
|
||||
$(PIPRUN) pip install -e .[$*] -c $(CONSTRAINTS_FILE)
|
||||
|
||||
# Prepare the development environment.
|
||||
# Build submodules.
|
||||
# Install the pacakge in editable mode with all optional dependencies and pre-commit hook.
|
||||
init-qlib-env:
|
||||
# note: You may need to install torch manually
|
||||
# todo: downgrade ruamel.yaml in pyqlib
|
||||
conda create -n qlibRDAgent python=3.8 -y
|
||||
@source $$(conda info --base)/etc/profile.d/conda.sh && conda activate qlibRDAgent && which pip && pip install pyqlib && pip install ruamel-yaml==0.17.21 && pip install torch==2.1.1 && pip install catboost==0.24.3 && conda deactivate
|
||||
|
||||
dev:
|
||||
$(PIPRUN) pip install -U pip setuptools wheel
|
||||
$(PIPRUN) pip install -e .[docs,lint,package,test] -c $(CONSTRAINTS_FILE)
|
||||
$(PIPRUN) pip install -U kaggle
|
||||
if [ "$(CI)" != "true" ] && command -v pre-commit > /dev/null 2>&1; then pre-commit install --hook-type pre-push; fi
|
||||
|
||||
# Generate constraints for current Python version.
|
||||
constraints: deepclean
|
||||
$(PIPRUN) --python $(PYTHON_VERSION) pip install --upgrade -e .[docs,lint,package,test]
|
||||
$(PIPRUN) pip freeze --exclude-editable > $(CONSTRAINTS_FILE)
|
||||
|
||||
########################################################################################
|
||||
# Lint and pre-commit
|
||||
########################################################################################
|
||||
|
||||
# Check lint with black.
|
||||
black:
|
||||
$(PIPRUN) python -m black --check --diff . --extend-exclude "(test/scripts|test/notebook/testfiles|git_ignore_folder|web)" -l 120
|
||||
|
||||
# Check lint with isort.
|
||||
isort:
|
||||
$(PIPRUN) python -m isort --check . -s git_ignore_folder -s test/scripts -s test/notebook/testfiles -s web
|
||||
|
||||
# Check lint with mypy.
|
||||
# First deal with the core folder, and then gradually increase the scope of detection,
|
||||
# and eventually realize the detection of the complete project.
|
||||
mypy:
|
||||
$(PIPRUN) python -m mypy rdagent/core
|
||||
|
||||
# Check lint with ruff.
|
||||
# First deal with the core folder, and then gradually increase the scope of detection,
|
||||
# and eventually realize the detection of the complete project.
|
||||
ruff:
|
||||
$(PIPRUN) ruff check rdagent/core --ignore FBT001,FBT002,I001,E501 # --exclude rdagent/scripts,git_ignore_folder
|
||||
|
||||
# Check lint with toml-sort.
|
||||
toml-sort:
|
||||
$(PIPRUN) toml-sort --check pyproject.toml
|
||||
|
||||
# Check lint with all linters.
|
||||
# Prioritize fixing isort, then black, otherwise you'll get weird and unfixable black errors.
|
||||
# lint: mypy ruff
|
||||
lint: mypy ruff isort black toml-sort
|
||||
|
||||
# Run pre-commit with autofix against all files.
|
||||
pre-commit:
|
||||
pre-commit run --all-files
|
||||
|
||||
########################################################################################
|
||||
# Auto Lint
|
||||
########################################################################################
|
||||
|
||||
# Auto lint with black.
|
||||
auto-black:
|
||||
$(PIPRUN) python -m black . --extend-exclude "(test/scripts|test/notebook/testfiles|git_ignore_folder|.venv|web)" -l 120
|
||||
|
||||
# Auto lint with isort.
|
||||
auto-isort:
|
||||
$(PIPRUN) python -m isort . -s git_ignore_folder -s test/scripts -s test/notebook/testfiles -s .venv -s web
|
||||
|
||||
# Auto lint with toml-sort.
|
||||
auto-toml-sort:
|
||||
$(PIPRUN) toml-sort pyproject.toml
|
||||
|
||||
# Auto lint with all linters.
|
||||
auto-lint: auto-isort auto-black auto-toml-sort
|
||||
|
||||
########################################################################################
|
||||
# Test
|
||||
########################################################################################
|
||||
|
||||
# Clean and run test with coverage.
|
||||
test-run:
|
||||
$(PIPRUN) python -m coverage erase
|
||||
$(PIPRUN) python -m coverage run --concurrency=multiprocessing -m pytest --ignore test/scripts
|
||||
$(PIPRUN) python -m coverage combine
|
||||
|
||||
test-run-offline:
|
||||
# some test that does not require api calling
|
||||
$(PIPRUN) python -m coverage erase
|
||||
$(PIPRUN) python -m coverage run --concurrency=multiprocessing -m pytest -m "offline" --ignore test/scripts
|
||||
$(PIPRUN) python -m coverage combine
|
||||
|
||||
# Generate coverage report for terminal and xml.
|
||||
# TODO: we may have higher coverage rate if we have more test
|
||||
test: test-run
|
||||
$(PIPRUN) python -m coverage report --fail-under 20 # 80
|
||||
$(PIPRUN) python -m coverage xml --fail-under 20 # 80
|
||||
|
||||
test-offline: test-run-offline
|
||||
$(PIPRUN) python -m coverage report --fail-under 20 # 80
|
||||
$(PIPRUN) python -m coverage xml --fail-under 20 # 80
|
||||
|
||||
########################################################################################
|
||||
# Package
|
||||
########################################################################################
|
||||
|
||||
# Build the package.
|
||||
build:
|
||||
$(PIPRUN) python -m build
|
||||
|
||||
# Upload the package.
|
||||
upload:
|
||||
$(PIPRUN) python -m twine upload dist/*
|
||||
|
||||
########################################################################################
|
||||
# Documentation
|
||||
########################################################################################
|
||||
|
||||
# Generate documentation with auto build when changes happen.
|
||||
docs-autobuild:
|
||||
$(PIPRUN) python -m sphinx_autobuild docs $(PUBLIC_DIR) \
|
||||
--watch README.md \
|
||||
--watch rdagent
|
||||
|
||||
# Generate changelog from git commits.
|
||||
# The -c and -s arguments should match
|
||||
# If -c uses Basic (default, inherits from base class), -s optional argument: # If -c uses conventional (inherits from base class), -s optional parameter: add,fix,change,remove,merge,doc
|
||||
# If -c uses conventional (inherits from base class), -s is optional: build,chore,ci,deps,doc,docs,feat,fix,perf,ref,refactor,revert,style,test,tests
|
||||
# If -c uses angular (inherits from conventional), -s optional argument: build,chore,ci,deps,doc,docs,feat,fix,perf,ref,refactor,revert,style,test,tests
|
||||
# NOTE(xuan.hu): Need to be run before document generation to take effect.
|
||||
# $(PIPRUN) git-changelog -ETrio $(CHANGELOG_PATH) -c conventional -s build,chore,ci,docs,feat,fix,perf,refactor,revert,style,test
|
||||
changelog:
|
||||
@if wget -q --spider $(CHANGELOG_URL); then \
|
||||
echo "Existing Changelog found at '$(CHANGELOG_URL)', download for incremental generation."; \
|
||||
wget -q -O $(CHANGELOG_PATH) $(CHANGELOG_URL); \
|
||||
fi
|
||||
$(PIPRUN) LATEST_TAG=$$(git tag --sort=-creatordate | head -n 1); \
|
||||
git-changelog --bump $$LATEST_TAG -Tio docs/changelog.md -c conventional -s build,chore,ci,deps,doc,docs,feat,fix,perf,ref,refactor,revert,style,test,tests
|
||||
|
||||
# Generate release notes from changelog.
|
||||
release-notes:
|
||||
@$(PIPRUN) git-changelog --input $(CHANGELOG_PATH) --release-notes
|
||||
|
||||
# Build documentation only from rdagent.
|
||||
docs-gen:
|
||||
$(PIPRUN) python -m sphinx.cmd.build -W docs $(PUBLIC_DIR)
|
||||
|
||||
# Generate mypy reports.
|
||||
docs-mypy: docs-gen
|
||||
$(PIPRUN) python -m mypy rdagent test --exclude git_ignore_folder --exclude rdagent/scripts --html-report $(PUBLIC_DIR)/reports/mypy
|
||||
|
||||
# Generate html coverage reports with badge.
|
||||
docs-coverage: test-run docs-gen
|
||||
$(PIPRUN) python -m coverage html -d $(PUBLIC_DIR)/reports/coverage --fail-under 80
|
||||
$(PIPRUN) bash scripts/generate-coverage-badge.sh $(PUBLIC_DIR)/_static/badges
|
||||
|
||||
# Generate all documentation with reports.
|
||||
docs: changelog docs-gen docs-mypy docs-coverage
|
||||
|
||||
|
||||
########################################################################################
|
||||
# End
|
||||
########################################################################################
|
||||
@@ -0,0 +1,577 @@
|
||||
<h4 align="center">
|
||||
<img src="docs/_static/logo.png" alt="RA-Agent logo" style="width:70%; ">
|
||||
|
||||
<a href="https://rdagent.azurewebsites.net" target="_blank">🖥️ Live Demo</a> |
|
||||
<a href="https://rdagent.azurewebsites.net/factor_loop" target="_blank">🎥 Demo Video</a> <a href="https://www.youtube.com/watch?v=JJ4JYO3HscM&list=PLALmKB0_N3_i52fhUmPQiL4jsO354uopR" target="_blank">▶️YouTube</a> |
|
||||
<a href="https://rdagent.readthedocs.io/en/latest/index.html" target="_blank">📖 Documentation</a> |
|
||||
<a href="https://aka.ms/RD-Agent-Tech-Report" target="_blank">📄 Tech Report</a> |
|
||||
<a href="#-paperwork-list"> 📃 Papers </a>
|
||||
</h3>
|
||||
|
||||
|
||||
[](https://github.com/microsoft/RD-Agent/actions/workflows/ci.yml)
|
||||
[](https://github.com/microsoft/RD-Agent/actions/workflows/github-code-scanning/codeql)
|
||||
[](https://github.com/microsoft/RD-Agent/actions/workflows/dependabot/dependabot-updates)
|
||||
[](https://github.com/microsoft/RD-Agent/actions/workflows/pr.yml)
|
||||
[](https://github.com/microsoft/RD-Agent/actions/workflows/release.yml)
|
||||
[](https://pypi.org/project/rdagent/#files)
|
||||
[](https://pypi.org/project/rdagent/)
|
||||
[](https://pypi.org/project/rdagent/)
|
||||
[](https://github.com/microsoft/RD-Agent/releases)
|
||||
[](https://github.com/microsoft/RD-Agent/blob/main/LICENSE)
|
||||
[](https://github.com/pre-commit/pre-commit)
|
||||
[](http://mypy-lang.org/)
|
||||
[](https://github.com/astral-sh/ruff)
|
||||
[](https://discord.gg/ybQ97B6Jjy)
|
||||
[](https://rdagent.readthedocs.io/en/latest/?badge=latest)
|
||||
[](https://github.com/microsoft/RD-Agent/actions/workflows/readthedocs-preview.yml) <!-- this badge is too long, please place it in the last one to make it pretty -->
|
||||
[](https://arxiv.org/abs/2505.14738)
|
||||
|
||||
|
||||
# 📰 News
|
||||
| 🗞️ News | 📝 Description |
|
||||
| -- | ------ |
|
||||
| ICML 2026 Acceptance | We are thrilled to announce that our paper [FT-Dojo: Towards Autonomous LLM Fine-Tuning with Language Agents](https://arxiv.org/abs/2603.01712) has been accepted to ICML 2026. The FT-Agent implementation is available in the [LLM fine-tuning guide](rdagent/app/finetune/llm/README.md). |
|
||||
| ACL 2026 Findings Acceptance | We are thrilled to announce that our paper [Reasoning as Gradient](https://arxiv.org/abs/2603.01692) has been accepted to ACL 2026 Findings. Execution traces are available at [Gome GPT-5 Traces](https://huggingface.co/datasets/amstrongzyf/Gome-GPT5-Traces) |
|
||||
| Web UI Release | We release a new frontend that can be built and served by `rdagent server_ui` for real-time interaction and trace viewing, currently excluding the `data_science` scenario. |
|
||||
| NeurIPS 2025 Acceptance | We are thrilled to announce that our paper [R&D-Agent-Quant](https://arxiv.org/abs/2505.15155) has been accepted to NeurIPS 2025 |
|
||||
| [Technical Report Release](#overall-technical-report) | Overall framework description and results on MLE-bench |
|
||||
| [R&D-Agent-Quant Release](#deep-application-in-diverse-scenarios) | Apply R&D-Agent to quant trading |
|
||||
| MLE-Bench Results Released | R&D-Agent currently leads as the [top-performing machine learning engineering agent](#-the-best-machine-learning-engineering-agent) on MLE-bench |
|
||||
| Support LiteLLM Backend | We now fully support **[LiteLLM](https://github.com/BerriAI/litellm)** as our default backend for integration with multiple LLM providers. |
|
||||
| General Data Science Agent | [Data Science Agent](https://rdagent.readthedocs.io/en/latest/scens/data_science.html) |
|
||||
| Kaggle Scenario release | We release **[Kaggle Agent](https://rdagent.readthedocs.io/en/latest/scens/data_science.html)**, try the new features! |
|
||||
| Official WeChat group release | We created a WeChat group, welcome to join! (🗪[QR Code](https://github.com/microsoft/RD-Agent/issues/880)) |
|
||||
| Official Discord release | We launch our first chatting channel in Discord (🗪[](https://discord.gg/ybQ97B6Jjy)) |
|
||||
| First release | **R&D-Agent** is released on GitHub |
|
||||
|
||||
|
||||
|
||||
# 🏆 The Best Machine Learning Engineering Agent!
|
||||
|
||||
[MLE-bench](https://github.com/openai/mle-bench) is a comprehensive benchmark evaluating the performance of AI agents on machine learning engineering tasks. Utilizing datasets from 75 Kaggle competitions, MLE-bench provides robust assessments of AI systems' capabilities in real-world ML engineering scenarios.
|
||||
|
||||
R&D-Agent currently leads as the top-performing machine learning engineering agent on MLE-bench:
|
||||
|
||||
| Agent | Low == Lite (%) | Medium (%) | High (%) | All (%) |
|
||||
|---------|--------|-----------|---------|----------|
|
||||
| R&D-Agent o3(R)+GPT-4.1(D) | 51.52 ± 6.9 | 19.3 ± 5.5 | 26.67 ± 0 | 30.22 ± 1.5 |
|
||||
| R&D-Agent o1-preview | 48.18 ± 2.49 | 8.95 ± 2.36 | 18.67 ± 2.98 | 22.4 ± 1.1 |
|
||||
| AIDE o1-preview | 34.3 ± 2.4 | 8.8 ± 1.1 | 10.0 ± 1.9 | 16.9 ± 1.1 |
|
||||
|
||||
**Notes:**
|
||||
- **O3(R)+GPT-4.1(D)**: This version is designed to both reduce average time per loop and leverage a cost-effective combination of backend LLMs by seamlessly integrating Research Agent (o3) with Development Agent (GPT-4.1).
|
||||
- **AIDE o1-preview**: Represents the previously best public result on MLE-bench as reported in the original MLE-bench paper.
|
||||
- Average and standard deviation results for R&D-Agent o1-preview is based on a independent of 5 seeds and for R&D-Agent o3(R)+GPT-4.1(D) is based on 6 seeds.
|
||||
- According to MLE-Bench, the 75 competitions are categorized into three levels of complexity: **Low==Lite** if we estimate that an experienced ML engineer can produce a sensible solution in under 2 hours, excluding the time taken to train any models; **Medium** if it takes between 2 and 10 hours; and **High** if it takes more than 10 hours.
|
||||
|
||||
You can inspect the detailed runs of the above results online.
|
||||
- [R&D-Agent o1-preview detailed runs](https://aka.ms/RD-Agent_MLE-Bench_O1-preview)
|
||||
- [R&D-Agent o3(R)+GPT-4.1(D) detailed runs](https://aka.ms/RD-Agent_MLE-Bench_O3_GPT41)
|
||||
|
||||
For running R&D-Agent on MLE-bench, refer to **[MLE-bench Guide: Running ML Engineering via MLE-bench](https://rdagent.readthedocs.io/en/latest/scens/data_science.html)**
|
||||
|
||||
# 🥇 The First Data-Centric Quant Multi-Agent Framework!
|
||||
|
||||
R&D-Agent for Quantitative Finance, in short **RD-Agent(Q)**, is the first data-centric, multi-agent framework designed to automate the full-stack research and development of quantitative strategies via coordinated factor-model co-optimization.
|
||||
|
||||
|
||||
|
||||
Extensive experiments in real stock markets show that, at a cost under $10, RD-Agent(Q) achieves approximately 2× higher ARR than benchmark factor libraries while using over 70% fewer factors. It also surpasses state-of-the-art deep time-series models under smaller resource budgets. Its alternating factor–model optimization further delivers excellent trade-off between predictive accuracy and strategy robustness.
|
||||
|
||||
You can learn more details about **RD-Agent(Q)** through the [paper](https://arxiv.org/abs/2505.15155) and reproduce it through the [documentation](https://rdagent.readthedocs.io/en/latest/scens/quant_agent_fin.html).
|
||||
|
||||
# Data Science Agent Preview
|
||||
Check out our demo video showcasing the current progress of our Data Science Agent under development:
|
||||
|
||||
https://github.com/user-attachments/assets/3eccbecb-34a4-4c81-bce4-d3f8862f7305
|
||||
|
||||
# 🌟 Introduction
|
||||
<div align="center">
|
||||
<img src="docs/_static/scen.png" alt="Our focused scenario" style="width:80%; ">
|
||||
</div>
|
||||
|
||||
R&D-Agent aims to automate the most critical and valuable aspects of the industrial R&D process, and we begin with focusing on the data-driven scenarios to streamline the development of models and data.
|
||||
Methodologically, we have identified a framework with two key components: 'R' for proposing new ideas and 'D' for implementing them.
|
||||
We believe that the automatic evolution of R&D will lead to solutions of significant industrial value.
|
||||
|
||||
|
||||
<!-- Tag Cloud -->
|
||||
R&D is a very general scenario. The advent of R&D-Agent can be your
|
||||
- 💰 **Automatic Quant Factory** ([🎥Demo Video](https://rdagent.azurewebsites.net/factor_loop)|[▶️YouTube](https://www.youtube.com/watch?v=X4DK2QZKaKY&t=6s))
|
||||
- 🤖 **Data Mining Agent:** Iteratively proposing data & models ([🎥Demo Video 1](https://rdagent.azurewebsites.net/model_loop)|[▶️YouTube](https://www.youtube.com/watch?v=dm0dWL49Bc0&t=104s)) ([🎥Demo Video 2](https://rdagent.azurewebsites.net/dmm)|[▶️YouTube](https://www.youtube.com/watch?v=VIaSTZuoZg4)) and implementing them by gaining knowledge from data.
|
||||
- 🦾 **Research Copilot:** Auto read research papers ([🎥Demo Video](https://rdagent.azurewebsites.net/report_model)|[▶️YouTube](https://www.youtube.com/watch?v=BiA2SfdKQ7o)) / financial reports ([🎥Demo Video](https://rdagent.azurewebsites.net/report_factor)|[▶️YouTube](https://www.youtube.com/watch?v=ECLTXVcSx-c)) and implement model structures or building datasets.
|
||||
- 🤖 **Kaggle Agent:** Auto Model Tuning and Feature Engineering([🎥Demo Video Coming Soon...]()) and implementing them to achieve more in competitions.
|
||||
- 🧪 **FT-Agent:** Autonomous LLM fine-tuning for benchmark-driven domain adaptation. See the [LLM fine-tuning guide](rdagent/app/finetune/llm/README.md).
|
||||
- ...
|
||||
|
||||
You can click the links above to view the demo. We're continuously adding more methods and scenarios to the project to enhance your R&D processes and boost productivity.
|
||||
|
||||
Additionally, you can take a closer look at the examples in our **[🖥️ Live Demo](https://rdagent.azurewebsites.net/)**.
|
||||
|
||||
<div align="center">
|
||||
<a href="https://rdagent.azurewebsites.net/" target="_blank">
|
||||
<img src="docs/_static/demo.png" alt="Watch the demo" width="80%">
|
||||
</a>
|
||||
</div>
|
||||
|
||||
|
||||
# ⚡ Quick start
|
||||
|
||||
### RD-Agent currently only supports Linux.
|
||||
|
||||
You can try above demos by running the following command:
|
||||
|
||||
### 🐳 Docker installation.
|
||||
Users must ensure Docker is installed before attempting most scenarios. Please refer to the [official 🐳Docker page](https://docs.docker.com/engine/install/) for installation instructions.
|
||||
Ensure the current user can run Docker commands **without using sudo**. You can verify this by executing `docker run hello-world`.
|
||||
|
||||
### 🐍 Create a Conda Environment
|
||||
- Create a new conda environment with Python (3.10 and 3.11 are well-tested in our CI):
|
||||
```sh
|
||||
conda create -n rdagent python=3.10
|
||||
```
|
||||
- Activate the environment:
|
||||
```sh
|
||||
conda activate rdagent
|
||||
```
|
||||
|
||||
### 🛠️ Install the R&D-Agent
|
||||
|
||||
#### For Users
|
||||
- You can directly install the R&D-Agent package from PyPI:
|
||||
```sh
|
||||
pip install rdagent
|
||||
```
|
||||
|
||||
#### For Developers
|
||||
- If you want to try the latest version or contribute to RD-Agent, you can install it from the source and follow the development setup:
|
||||
```sh
|
||||
git clone https://github.com/microsoft/RD-Agent
|
||||
cd RD-Agent
|
||||
make dev
|
||||
```
|
||||
|
||||
More details can be found in the [development setup](https://rdagent.readthedocs.io/en/latest/development.html).
|
||||
|
||||
### 💊 Health check
|
||||
- rdagent provides a health check that currently checks two things.
|
||||
- whether the docker installation was successful.
|
||||
- whether the default port used by the [rdagent ui](https://github.com/microsoft/RD-Agent?tab=readme-ov-file#%EF%B8%8F-monitor-the-application-results) is occupied.
|
||||
```sh
|
||||
rdagent health_check --no-check-env
|
||||
```
|
||||
|
||||
|
||||
### ⚙️ Configuration
|
||||
- The demos requires following ability:
|
||||
- ChatCompletion
|
||||
- json_mode
|
||||
- embedding query
|
||||
|
||||
You can set your Chat Model and Embedding Model in the following ways:
|
||||
|
||||
> **🔥 Attention**: We now provide experimental support for **DeepSeek** models! You can use DeepSeek's official API for cost-effective and high-performance inference. See the configuration example below for DeepSeek setup.
|
||||
|
||||
- **Using LiteLLM (Default)**: We now support LiteLLM as a backend for integration with multiple LLM providers. You can configure in multiple ways:
|
||||
|
||||
**Option 1: Unified API base for both models**
|
||||
|
||||
*Configuration Example: `OpenAI` Setup :*
|
||||
|
||||
```bash
|
||||
cat << EOF > .env
|
||||
# Set to any model supported by LiteLLM.
|
||||
CHAT_MODEL=gpt-4o
|
||||
EMBEDDING_MODEL=text-embedding-3-small
|
||||
# Configure unified API base
|
||||
OPENAI_API_BASE=<your_unified_api_base>
|
||||
OPENAI_API_KEY=<replace_with_your_openai_api_key>
|
||||
```
|
||||
|
||||
*Configuration Example: `Azure OpenAI` Setup :*
|
||||
|
||||
> Before using this configuration, please confirm in advance that your `Azure OpenAI API key` supports `embedded models`.
|
||||
|
||||
```bash
|
||||
cat << EOF > .env
|
||||
EMBEDDING_MODEL=azure/<Model deployment supporting embedding>
|
||||
CHAT_MODEL=azure/<your deployment name>
|
||||
AZURE_API_KEY=<replace_with_your_openai_api_key>
|
||||
AZURE_API_BASE=<your_unified_api_base>
|
||||
AZURE_API_VERSION=<azure api version>
|
||||
```
|
||||
|
||||
**Option 2: Separate API bases for Chat and Embedding models**
|
||||
```bash
|
||||
cat << EOF > .env
|
||||
# Set to any model supported by LiteLLM.
|
||||
# Configure separate API bases for chat and embedding
|
||||
|
||||
# CHAT MODEL:
|
||||
CHAT_MODEL=gpt-4o
|
||||
OPENAI_API_BASE=<your_chat_api_base>
|
||||
OPENAI_API_KEY=<replace_with_your_openai_api_key>
|
||||
|
||||
# EMBEDDING MODEL:
|
||||
# TAKE siliconflow as an example, you can use other providers.
|
||||
# Note: embedding requires litellm_proxy prefix
|
||||
EMBEDDING_MODEL=litellm_proxy/BAAI/bge-large-en-v1.5
|
||||
LITELLM_PROXY_API_KEY=<replace_with_your_siliconflow_api_key>
|
||||
LITELLM_PROXY_API_BASE=https://api.siliconflow.cn/v1
|
||||
```
|
||||
|
||||
*Configuration Example: `DeepSeek` Setup :*
|
||||
|
||||
>Since many users encounter configuration errors when setting up DeepSeek. Here's a complete working example for DeepSeek Setup:
|
||||
```bash
|
||||
cat << EOF > .env
|
||||
# CHAT MODEL: Using DeepSeek Official API
|
||||
CHAT_MODEL=deepseek/deepseek-chat
|
||||
DEEPSEEK_API_KEY=<replace_with_your_deepseek_api_key>
|
||||
|
||||
# EMBEDDING MODEL: Using SiliconFlow for embedding since deepseek has no embedding model.
|
||||
# Note: embedding requires litellm_proxy prefix
|
||||
EMBEDDING_MODEL=litellm_proxy/BAAI/bge-m3
|
||||
LITELLM_PROXY_API_KEY=<replace_with_your_siliconflow_api_key>
|
||||
LITELLM_PROXY_API_BASE=https://api.siliconflow.cn/v1
|
||||
```
|
||||
|
||||
Notice: If you are using reasoning models that include thought processes in their responses (such as \<think> tags), you need to set the following environment variable:
|
||||
```bash
|
||||
REASONING_THINK_RM=True
|
||||
```
|
||||
|
||||
You can also use a deprecated backend if you only use `OpenAI API` or `Azure OpenAI` directly. For this deprecated setting and more configuration information, please refer to the [documentation](https://rdagent.readthedocs.io/en/latest/installation_and_configuration.html).
|
||||
|
||||
|
||||
|
||||
- If your environment configuration is complete, please execute the following commands to check if your configuration is valid. This step is necessary.
|
||||
|
||||
```bash
|
||||
rdagent health_check
|
||||
```
|
||||
|
||||
### 🚀 Run the Application
|
||||
|
||||
The **[🖥️ Live Demo](https://rdagent.azurewebsites.net/)** is implemented by the following commands(each item represents one demo, you can select the one you prefer):
|
||||
|
||||
- Run the **Automated Quantitative Trading & Iterative Factors Model Joint Evolution**: [Qlib](http://github.com/microsoft/qlib) self-loop factor & model proposal and implementation application
|
||||
```sh
|
||||
rdagent fin_quant
|
||||
```
|
||||
|
||||
- Run the **Automated Quantitative Trading & Iterative Factors Evolution**: [Qlib](http://github.com/microsoft/qlib) self-loop factor proposal and implementation application
|
||||
```sh
|
||||
rdagent fin_factor
|
||||
```
|
||||
|
||||
- Run the **Automated Quantitative Trading & Iterative Model Evolution**: [Qlib](http://github.com/microsoft/qlib) self-loop model proposal and implementation application
|
||||
```sh
|
||||
rdagent fin_model
|
||||
```
|
||||
|
||||
- Run the **Automated Quantitative Trading & Factors Extraction from Financial Reports**: Run the [Qlib](http://github.com/microsoft/qlib) factor extraction and implementation application based on financial reports
|
||||
```sh
|
||||
# 1. Generally, you can run this scenario using the following command:
|
||||
rdagent fin_factor_report --report-folder=<Your financial reports folder path>
|
||||
|
||||
# 2. Specifically, you need to prepare some financial reports first. You can follow this concrete example:
|
||||
wget https://github.com/SunsetWolf/rdagent_resource/releases/download/reports/all_reports.zip
|
||||
unzip all_reports.zip -d git_ignore_folder/reports
|
||||
rdagent fin_factor_report --report-folder=git_ignore_folder/reports
|
||||
```
|
||||
|
||||
- Run the **Automated Model Research & Development Copilot**: model extraction and implementation application
|
||||
```sh
|
||||
# 1. Generally, you can run your own papers/reports with the following command:
|
||||
rdagent general_model <Your paper URL>
|
||||
|
||||
# 2. Specifically, you can do it like this. For more details and additional paper examples, use `rdagent general_model -h`:
|
||||
rdagent general_model "https://arxiv.org/pdf/2210.09789"
|
||||
```
|
||||
|
||||
- Run the **Automated Medical Prediction Model Evolution**: Medical self-loop model proposal and implementation application
|
||||
|
||||
```bash
|
||||
# Generally, you can run the data science program with the following command:
|
||||
rdagent data_science --competition <your competition name>
|
||||
|
||||
# Specifically, you need to create a folder for storing competition files (e.g., competition description file, competition datasets, etc.), and configure the path to the folder in your environment. In addition, you need to use chromedriver when you download the competition descriptors, which you can follow for this specific example:
|
||||
|
||||
# 1. Download the dataset, extract it to the target folder.
|
||||
wget https://github.com/SunsetWolf/rdagent_resource/releases/download/ds_data/arf-12-hours-prediction-task.zip
|
||||
unzip arf-12-hours-prediction-task.zip -d ./git_ignore_folder/ds_data/
|
||||
|
||||
# 2. Configure environment variables in the `.env` file
|
||||
dotenv set DS_LOCAL_DATA_PATH "$(pwd)/git_ignore_folder/ds_data"
|
||||
dotenv set DS_CODER_ON_WHOLE_PIPELINE True
|
||||
dotenv set DS_IF_USING_MLE_DATA False
|
||||
dotenv set DS_SAMPLE_DATA_BY_LLM False
|
||||
dotenv set DS_SCEN rdagent.scenarios.data_science.scen.DataScienceScen
|
||||
|
||||
# 3. run the application
|
||||
rdagent data_science --competition arf-12-hours-prediction-task
|
||||
```
|
||||
|
||||
**NOTE:** For more information about the dataset, please refer to the [documentation](https://rdagent.readthedocs.io/en/latest/scens/data_science.html).
|
||||
|
||||
- Run the **Automated Kaggle Model Tuning & Feature Engineering**: self-loop model proposal and feature engineering implementation application <br />
|
||||
> Using **tabular-playground-series-dec-2021** as an example. <br />
|
||||
> 1. Register and login on the [Kaggle](https://www.kaggle.com/) website. <br />
|
||||
> 2. Configuring the Kaggle API. <br />
|
||||
> (1) Click on the avatar (usually in the top right corner of the page) -> `Settings` -> `Create New Token`, A file called `kaggle.json` will be downloaded. <br />
|
||||
> (2) Move `kaggle.json` to `~/.config/kaggle/` <br />
|
||||
> (3) Modify the permissions of the kaggle.json file. Reference command: `chmod 600 ~/.config/kaggle/kaggle.json` <br />
|
||||
> 3. Join the competition: Click `Join the competition` -> `I Understand and Accept` at the bottom of the [competition details page](https://www.kaggle.com/competitions/tabular-playground-series-dec-2021/data).
|
||||
```bash
|
||||
# Generally, you can run the Kaggle competition program with the following command:
|
||||
rdagent data_science --competition <your competition name>
|
||||
|
||||
# 1. Configure environment variables in the `.env` file
|
||||
mkdir -p ./git_ignore_folder/ds_data
|
||||
dotenv set DS_LOCAL_DATA_PATH "$(pwd)/git_ignore_folder/ds_data"
|
||||
dotenv set DS_CODER_ON_WHOLE_PIPELINE True
|
||||
dotenv set DS_IF_USING_MLE_DATA True
|
||||
dotenv set DS_SAMPLE_DATA_BY_LLM True
|
||||
dotenv set DS_SCEN rdagent.scenarios.data_science.scen.KaggleScen
|
||||
|
||||
# 2. run the application
|
||||
rdagent data_science --competition tabular-playground-series-dec-2021
|
||||
```
|
||||
|
||||
- Run **FT-Agent for Autonomous LLM Fine-Tuning**: an ICML 2026 LLM fine-tuning scenario for benchmark-driven data processing, training, evaluation, and feedback-guided refinement.
|
||||
```bash
|
||||
# See the full setup, benchmark descriptions, dataset notes, and examples:
|
||||
# rdagent/app/finetune/llm/README.md
|
||||
# Configure FT_TARGET_BENCHMARK and FT_BENCHMARK_DESCRIPTION before running.
|
||||
rdagent llm_finetune --base-model Qwen/Qwen2.5-7B-Instruct
|
||||
```
|
||||
|
||||
### 🖥️ Monitor the Application Results
|
||||
#### Streamlit UI
|
||||
|
||||
Use the Streamlit UI to view run logs, especially for the `data_science` scenario.
|
||||
|
||||
```sh
|
||||
rdagent ui --port 19899 --log-dir <your log folder like "log/"> --data-science
|
||||
```
|
||||
|
||||
About the `data_science` parameter: If you want to see the logs of the data science scenario, set the `data_science` parameter to `True`; otherwise set it to `False`.
|
||||
|
||||
#### Web UI
|
||||
|
||||
We also provide a separate web frontend in `web/` for the Flask backend started by `server_ui`.
|
||||
|
||||
**NOTE:** This web UI is different from `rdagent ui`. The current web UI does not support the `data_science` scenario yet. For the `data_science` scenario, please continue to use `rdagent ui --data-science`.
|
||||
|
||||
```sh
|
||||
cd web
|
||||
npm install
|
||||
```
|
||||
|
||||
To build the frontend for the Flask backend, generate the static assets into the default directory used by `server_ui`:
|
||||
|
||||
```sh
|
||||
cd web
|
||||
npm run build:flask
|
||||
```
|
||||
|
||||
By default, `server_ui` serves static files from `./git_ignore_folder/static`. If you need a different location, set the `UI_STATIC_PATH` environment variable before starting the backend.
|
||||
|
||||
Start the Flask backend and serve the built frontend together with the real-time APIs:
|
||||
|
||||
```sh
|
||||
rdagent server_ui --port 19899
|
||||
```
|
||||
|
||||
After that, open `http://127.0.0.1:19899` in your browser.
|
||||
|
||||
#### Common Notes
|
||||
|
||||
Port `19899` is used in the examples above. Before starting either UI, check whether this port is already occupied. If it is, please change it to another available port.
|
||||
|
||||
You can check whether the port is occupied by running:
|
||||
|
||||
```sh
|
||||
rdagent health_check --no-check-env --no-check-docker
|
||||
```
|
||||
|
||||
# 🏭 Scenarios
|
||||
|
||||
We have applied R&D-Agent to multiple valuable data-driven industrial scenarios.
|
||||
|
||||
|
||||
## 🎯 Goal: Agent for Data-driven R&D
|
||||
|
||||
In this project, we are aiming to build an Agent to automate Data-Driven R\&D that can
|
||||
+ 📄 Read real-world material (reports, papers, etc.) and **extract** key formulas, descriptions of interested **features** and **models**, which are the key components of data-driven R&D .
|
||||
+ 🛠️ **Implement** the extracted formulas (e.g., features, factors, and models) in runnable codes.
|
||||
+ Due to the limited ability of LLM in implementing at once, build an evolving process for the agent to improve performance by learning from feedback and knowledge.
|
||||
+ 💡 Propose **new ideas** based on current knowledge and observations.
|
||||
|
||||
<!--  -->
|
||||
|
||||
## 📈 Scenarios/Demos
|
||||
|
||||
In the two key areas of data-driven scenarios, model implementation and data building, our system aims to serve two main roles: 🦾Copilot and 🤖Agent.
|
||||
- The 🦾Copilot follows human instructions to automate repetitive tasks.
|
||||
- The 🤖Agent, being more autonomous, actively proposes ideas for better results in the future.
|
||||
|
||||
The supported scenarios are listed below:
|
||||
|
||||
| Scenario/Target | Model Implementation | Data Building |
|
||||
| -- | -- | -- |
|
||||
| **💹 Finance** | 🤖 [Iteratively Proposing Ideas & Evolving](https://rdagent.azurewebsites.net/model_loop)[▶️YouTube](https://www.youtube.com/watch?v=dm0dWL49Bc0&t=104s) | 🤖 [Iteratively Proposing Ideas & Evolving](https://rdagent.azurewebsites.net/factor_loop) [▶️YouTube](https://www.youtube.com/watch?v=X4DK2QZKaKY&t=6s) <br/> 🦾 [Auto reports reading & implementation](https://rdagent.azurewebsites.net/report_factor)[▶️YouTube](https://www.youtube.com/watch?v=ECLTXVcSx-c) |
|
||||
| **🩺 Medical** | 🤖 [Iteratively Proposing Ideas & Evolving](https://rdagent.azurewebsites.net/dmm)[▶️YouTube](https://www.youtube.com/watch?v=VIaSTZuoZg4) | - |
|
||||
| **🏭 General** | 🦾 [Auto paper reading & implementation](https://rdagent.azurewebsites.net/report_model)[▶️YouTube](https://www.youtube.com/watch?v=BiA2SfdKQ7o) <br/> 🤖 Auto Kaggle Model Tuning | 🤖Auto Kaggle feature Engineering |
|
||||
|
||||
- **[RoadMap](https://rdagent.readthedocs.io/en/latest/scens/data_science.html#roadmap)**: Currently, we are working hard to add new features to the Kaggle scenario.
|
||||
|
||||
Different scenarios vary in entrance and configuration. Please check the detailed setup tutorial in the scenarios documents.
|
||||
|
||||
Here is a gallery of [successful explorations](https://github.com/SunsetWolf/rdagent_resource/releases/download/demo_traces/demo_traces.zip) (5 traces showed in **[🖥️ Live Demo](https://rdagent.azurewebsites.net/)**). You can download and view the execution trace using [this command](https://github.com/microsoft/RD-Agent?tab=readme-ov-file#%EF%B8%8F-monitor-the-application-results) from the documentation.
|
||||
|
||||
Please refer to **[📖readthedocs_scen](https://rdagent.readthedocs.io/en/latest/scens/catalog.html)** for more details of the scenarios.
|
||||
|
||||
# ⚙️ Framework
|
||||
|
||||
<div align="center">
|
||||
<img src="docs/_static/Framework-RDAgent.png" alt="Framework-RDAgent" width="85%">
|
||||
</div>
|
||||
|
||||
|
||||
Automating the R&D process in data science is a highly valuable yet underexplored area in industry. We propose a framework to push the boundaries of this important research field.
|
||||
|
||||
The research questions within this framework can be divided into three main categories:
|
||||
| Research Area | Paper/Work List |
|
||||
|--------------------|-----------------|
|
||||
| **Benchmark the R&D abilities** | [Benchmark](#benchmark) |
|
||||
| **Idea proposal:** Explore new ideas or refine existing ones | [Research](#research) |
|
||||
| **Ability to realize ideas:** Implement and execute ideas | [Development](#development) |
|
||||
|
||||
We believe that the key to delivering high-quality solutions lies in the ability to evolve R&D capabilities. Agents should learn like human experts, continuously improving their R&D skills.
|
||||
|
||||
More documents can be found in the **[📖 readthedocs](https://rdagent.readthedocs.io/)**.
|
||||
|
||||
# 📃 Paper/Work list
|
||||
|
||||
## Overall Technical Report
|
||||
- [R&D-Agent: An LLM-Agent Framework Towards Autonomous Data Science](https://arxiv.org/abs/2505.14738)
|
||||
```BibTeX
|
||||
@misc{yang2025rdagentllmagentframeworkautonomous,
|
||||
title={R&D-Agent: An LLM-Agent Framework Towards Autonomous Data Science},
|
||||
author={Xu Yang and Xiao Yang and Shikai Fang and Yifei Zhang and Jian Wang and Bowen Xian and Qizheng Li and Jingyuan Li and Minrui Xu and Yuante Li and Haoran Pan and Yuge Zhang and Weiqing Liu and Yelong Shen and Weizhu Chen and Jiang Bian},
|
||||
year={2025},
|
||||
eprint={2505.14738},
|
||||
archivePrefix={arXiv},
|
||||
primaryClass={cs.AI},
|
||||
url={https://arxiv.org/abs/2505.14738},
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
## 📊 Benchmark
|
||||
- [Towards Data-Centric Automatic R&D](https://arxiv.org/abs/2404.11276)
|
||||
```BibTeX
|
||||
@misc{chen2024datacentric,
|
||||
title={Towards Data-Centric Automatic R&D},
|
||||
author={Haotian Chen and Xinjie Shen and Zeqi Ye and Wenjun Feng and Haoxue Wang and Xiao Yang and Xu Yang and Weiqing Liu and Jiang Bian},
|
||||
year={2024},
|
||||
eprint={2404.11276},
|
||||
archivePrefix={arXiv},
|
||||
primaryClass={cs.AI}
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
## 🔍 Research
|
||||
|
||||
In a data mining expert's daily research and development process, they propose a hypothesis (e.g., a model structure like RNN can capture patterns in time-series data), design experiments (e.g., finance data contains time-series and we can verify the hypothesis in this scenario), implement the experiment as code (e.g., Pytorch model structure), and then execute the code to get feedback (e.g., metrics, loss curve, etc.). The experts learn from the feedback and improve in the next iteration.
|
||||
|
||||
Based on the principles above, we have established a basic method framework that continuously proposes hypotheses, verifies them, and gets feedback from the real-world practice. This is the first scientific research automation framework that supports linking with real-world verification.
|
||||
|
||||
For more detail, please refer to our **[🖥️ Live Demo page](https://rdagent.azurewebsites.net)**.
|
||||
|
||||
## 🛠️ Development
|
||||
|
||||
- [Collaborative Evolving Strategy for Automatic Data-Centric Development](https://arxiv.org/abs/2407.18690)
|
||||
```BibTeX
|
||||
@misc{yang2024collaborative,
|
||||
title={Collaborative Evolving Strategy for Automatic Data-Centric Development},
|
||||
author={Xu Yang and Haotian Chen and Wenjun Feng and Haoxue Wang and Zeqi Ye and Xinjie Shen and Xiao Yang and Shizhao Sun and Weiqing Liu and Jiang Bian},
|
||||
year={2024},
|
||||
eprint={2407.18690},
|
||||
archivePrefix={arXiv},
|
||||
primaryClass={cs.AI}
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
## Deep Application in Diverse Scenarios
|
||||
|
||||
- [FT-Dojo: Towards Autonomous LLM Fine-Tuning with Language Agents](https://arxiv.org/abs/2603.01712)
|
||||
|
||||
```BibTeX
|
||||
@misc{li2026ftdojo,
|
||||
title={FT-Dojo: Towards Autonomous LLM Fine-Tuning with Language Agents},
|
||||
author={Qizheng Li and Yifei Zhang and Xiao Yang and Xu Yang and Zhuo Wang and Weiqing Liu and Jiang Bian},
|
||||
year={2026},
|
||||
eprint={2603.01712},
|
||||
archivePrefix={arXiv},
|
||||
primaryClass={cs.AI},
|
||||
url={https://arxiv.org/abs/2603.01712}
|
||||
}
|
||||
```
|
||||
|
||||
FT-Agent, the autonomous LLM fine-tuning scenario from this paper, is available through the [LLM fine-tuning guide](rdagent/app/finetune/llm/README.md).
|
||||
|
||||
- [R&D-Agent-Quant: A Multi-Agent Framework for Data-Centric Factors and Model Joint Optimization](https://arxiv.org/abs/2505.15155)
|
||||
```BibTeX
|
||||
@misc{li2025rdagentquantmultiagentframeworkdatacentric,
|
||||
title={R&D-Agent-Quant: A Multi-Agent Framework for Data-Centric Factors and Model Joint Optimization},
|
||||
author={Yuante Li and Xu Yang and Xiao Yang and Minrui Xu and Xisen Wang and Weiqing Liu and Jiang Bian},
|
||||
year={2025},
|
||||
eprint={2505.15155},
|
||||
archivePrefix={arXiv},
|
||||
primaryClass={q-fin.CP},
|
||||
url={https://arxiv.org/abs/2505.15155},
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
- [Reasoning as Gradient: Scaling MLE Agents Beyond Tree Search](https://arxiv.org/abs/2603.01692)
|
||||
|
||||
```BibTeX
|
||||
@article{zhang2026reasoning,
|
||||
title={Reasoning as Gradient: Scaling MLE Agents Beyond Tree Search},
|
||||
author={Zhang, Yifei and Yang, Xu and Yang, Xiao and Xian, Bowen and Li, Qizheng and Fang, Shikai and Li, Jingyuan and Wang, Jian and Xu, Mingrui and Liu, Weiqing and others},
|
||||
journal={arXiv preprint arXiv:2603.01692},
|
||||
year={2026}
|
||||
}
|
||||
```
|
||||
|
||||
You can check the detailed execution traces online at [Gome GPT-5 Traces](https://huggingface.co/datasets/amstrongzyf/Gome-GPT5-Traces).
|
||||
|
||||
# 🤝 Contributing
|
||||
|
||||
We welcome contributions and suggestions to improve R&D-Agent. Please refer to the [Contributing Guide](CONTRIBUTING.md) for more details on how to contribute.
|
||||
|
||||
Before submitting a pull request, ensure that your code passes the automatic CI checks.
|
||||
|
||||
## 📝 Guidelines
|
||||
This project welcomes contributions and suggestions.
|
||||
Contributing to this project is straightforward and rewarding. Whether it's solving an issue, addressing a bug, enhancing documentation, or even correcting a typo, every contribution is valuable and helps improve R&D-Agent.
|
||||
|
||||
To get started, you can explore the issues list, or search for `TODO:` comments in the codebase by running the command `grep -r "TODO:"`.
|
||||
|
||||
<img src="https://img.shields.io/github/contributors-anon/microsoft/RD-Agent"/>
|
||||
|
||||
<a href="https://github.com/microsoft/RD-Agent/graphs/contributors">
|
||||
<img src="https://contrib.rocks/image?repo=microsoft/RD-Agent&max=100&columns=15" />
|
||||
</a>
|
||||
|
||||
Before we released R&D-Agent as an open-source project on GitHub, it was an internal project within our group. Unfortunately, the internal commit history was not preserved when we removed some confidential code. As a result, some contributions from our group members, including Haotian Chen, Wenjun Feng, Haoxue Wang, Zeqi Ye, Xinjie Shen, and Jinhui Li, were not included in the public commits.
|
||||
|
||||
# ⚖️ Legal disclaimer
|
||||
<p style="line-height: 1; font-style: italic;">The RD-agent is provided “as is”, without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose and noninfringement. The RD-agent is aimed to facilitate research and development process in the financial industry and not ready-to-use for any financial investment or advice. Users shall independently assess and test the risks of the RD-agent in a specific use scenario, ensure the responsible use of AI technology, including but not limited to developing and integrating risk mitigation measures, and comply with all applicable laws and regulations in all applicable jurisdictions. The RD-agent does not provide financial opinions or reflect the opinions of Microsoft, nor is it designed to replace the role of qualified financial professionals in formulating, assessing, and approving finance products. The inputs and outputs of the RD-agent belong to the users and users shall assume all liability under any theory of liability, whether in contract, torts, regulatory, negligence, products liability, or otherwise, associated with use of the RD-agent and any inputs and outputs thereof.</p>
|
||||
@@ -0,0 +1,7 @@
|
||||
# WeHub 来源说明
|
||||
|
||||
- 原始项目：`microsoft/RD-Agent`
|
||||
- 原始仓库：https://github.com/microsoft/RD-Agent
|
||||
- 导入方式：上游默认分支的最新快照
|
||||
- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准
|
||||
- 本文件仅用于记录来源，不代表 WeHub 是原项目作者
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
<!-- BEGIN MICROSOFT SECURITY.MD V0.0.9 BLOCK -->
|
||||
|
||||
## Security
|
||||
|
||||
Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet) and [Xamarin](https://github.com/xamarin).
|
||||
|
||||
If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/security.md/definition), please report it to us as described below.
|
||||
|
||||
## Reporting Security Issues
|
||||
|
||||
**Please do not report security vulnerabilities through public GitHub issues.**
|
||||
|
||||
Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/security.md/msrc/create-report).
|
||||
|
||||
If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/security.md/msrc/pgp).
|
||||
|
||||
You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc).
|
||||
|
||||
Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue:
|
||||
|
||||
* Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.)
|
||||
* Full paths of source file(s) related to the manifestation of the issue
|
||||
* The location of the affected source code (tag/branch/commit or direct URL)
|
||||
* Any special configuration required to reproduce the issue
|
||||
* Step-by-step instructions to reproduce the issue
|
||||
* Proof-of-concept or exploit code (if possible)
|
||||
* Impact of the issue, including how an attacker might exploit the issue
|
||||
|
||||
This information will help us triage your report more quickly.
|
||||
|
||||
If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/security.md/msrc/bounty) page for more details about our active programs.
|
||||
|
||||
## Preferred Languages
|
||||
|
||||
We prefer all communications to be in English.
|
||||
|
||||
## Policy
|
||||
|
||||
Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/security.md/cvd).
|
||||
|
||||
<!-- END MICROSOFT SECURITY.MD BLOCK -->
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
# TODO: The maintainer of this repo has not yet edited this file
|
||||
|
||||
**REPO OWNER**: Do you want Customer Service & Support (CSS) support for this product/project?
|
||||
|
||||
- **No CSS support:** Fill out this template with information about how to file issues and get help.
|
||||
- **Yes CSS support:** Fill out an intake form at [aka.ms/onboardsupport](https://aka.ms/onboardsupport). CSS will work with/help you to determine next steps.
|
||||
- **Not sure?** Fill out an intake as though the answer were "Yes". CSS will help you decide.
|
||||
|
||||
*Then remove this first heading from this SUPPORT.MD file before publishing your repo.*
|
||||
|
||||
# Support
|
||||
|
||||
## How to file issues and get help
|
||||
|
||||
This project uses GitHub Issues to track bugs and feature requests. Please search the existing
|
||||
issues before filing new issues to avoid duplicates. For new issues, file your bug or
|
||||
feature request as a new Issue.
|
||||
|
||||
For help and questions about using this project, please **REPO MAINTAINER: INSERT INSTRUCTIONS HERE
|
||||
FOR HOW TO ENGAGE REPO OWNERS OR COMMUNITY FOR HELP. COULD BE A STACK OVERFLOW TAG OR OTHER
|
||||
CHANNEL. WHERE WILL YOU HELP PEOPLE?**.
|
||||
|
||||
## Microsoft Support Policy
|
||||
|
||||
Support for this **PROJECT or PRODUCT** is limited to the resources listed above.
|
||||
@@ -0,0 +1,10 @@
|
||||
We encourage to set the TODOs in code. But some TODOs are more global.
|
||||
So we place it here.
|
||||
|
||||
|
||||
- [ ] Aligning the naming of files in components & scenarios.
|
||||
- We would like to have the same logic for naming convention in components(reusable components for all scenarios) and scenarios (componets for specific scenario).
|
||||
- But now we have following mismatch
|
||||
- `coder` in `components` & `developer` in `components`
|
||||
- [ ] The name of the folders mismatch with the content in them.
|
||||
- Why are scenarios in experiments?
|
||||
@@ -0,0 +1,5 @@
|
||||
azure-identity==1.17.1
|
||||
dill==0.3.9
|
||||
pillow==10.4.0
|
||||
psutil==6.1.0
|
||||
scipy==1.14.1
|
||||
@@ -0,0 +1,5 @@
|
||||
azure-identity==1.17.1
|
||||
dill==0.3.9
|
||||
pillow==10.4.0
|
||||
psutil==6.1.0
|
||||
scipy==1.14.1
|
||||
@@ -0,0 +1,20 @@
|
||||
# Minimal makefile for Sphinx documentation
|
||||
#
|
||||
|
||||
# You can set these variables from the command line, and also
|
||||
# from the environment for the first two.
|
||||
SPHINXOPTS ?=
|
||||
SPHINXBUILD ?= sphinx-build
|
||||
SOURCEDIR = .
|
||||
BUILDDIR = build
|
||||
|
||||
# Put it first so that "make" without argument is like "make help".
|
||||
help:
|
||||
@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
|
||||
|
||||
.PHONY: help Makefile
|
||||
|
||||
# Catch-all target: route all unknown targets to Sphinx using the new
|
||||
# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
|
||||
%: Makefile
|
||||
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
|
||||
Vendored
BIN
Binary file not shown.
|
After Width: | Height: | Size: 339 KiB |
Vendored
+332
@@ -0,0 +1,332 @@
|
||||
{
|
||||
"alpha053_15": {
|
||||
"description": "Reversal class factor, negative delta of a ratio involving close, low, and high prices over 15 days.",
|
||||
"formulation": "-1 times Deltaleft(frac{(text{close} - text{low}) - (text{high} - text{close})}{text{close} - text{low}}, 15right)",
|
||||
"variables": {
|
||||
"Delta(x, d)": "Change in 'x' over 'd' days.",
|
||||
"text{close}": "Closing price of the stock.",
|
||||
"text{low}": "Lowest price of the stock for the day.",
|
||||
"text{high}": "Highest price of the stock for the day."
|
||||
},
|
||||
"Category": "Volume&Price",
|
||||
"Difficulty": "Easy",
|
||||
"gt_code": "import pandas as pd\ndata_pv = pd.read_hdf('daily_pv.h5')\nnew_df= data_pv.reset_index()\n# Calculate Alpha053\nnew_df['ratio'] = (new_df['$close'] - new_df['$low'] - (new_df['$high'] - new_df['$close'])) / (new_df['$close'] - new_df['$low'])\n# the change of ratio in new_df over the 15 days\nnew_df['result']=-new_df['ratio'].diff(15)\n# transfer the result to series\nresult=pd.DataFrame(new_df['result']).set_index(data_pv.index)\nresult=result['result']\nresult.to_hdf('result.h5', key='data')"
|
||||
},
|
||||
"liquidity_imbalance": {
|
||||
"description": "liquidity_imbalance=std(minute trading liquidity_imbalance)/mean(minute trading liquidity_imbalance).",
|
||||
"formulation": "liquidity_imbalance = frac{text{std}(text{minute trading liquidity_imbalance})}{text{mean}(text{minute liquidity_imbalance})}",
|
||||
"variables": {
|
||||
"std(minute liquidity_imbalance)": "Standard deviation of trading liquidity_imbalance for each minute of the trading day.",
|
||||
"mean(minute liquidity_imbalance)": "Mean of trading liquidity_imbalance for each minute of the trading day.",
|
||||
"liquidity_imbalance": "(bid_size-ask_size)/(bid_size+ask_size), we use something like bidV for the size"
|
||||
},
|
||||
"Category": "High-Frequency",
|
||||
"Difficulty": "Medium",
|
||||
"gt_code": "import pandas as pd\ndata_hf = pd.read_hdf('high_freq.h5')\nsample_df= data_hf.reset_index()\n# Convert 'datetime' column to datetime and extract date for grouping\nsample_df['date'] = sample_df['datetime'].dt.date\nsample_df['liquidity_imbalance']=(sample_df['bidV']-sample_df['askV'])/(sample_df['bidV']+sample_df['askV'])\n# Group by instrument and date\ngrouped = sample_df.groupby(['date','instrument'])['liquidity_imbalance']\n# Calculate mean and standard deviation of the volume for each group\nstats = grouped.agg(['mean', 'std'])\n# Calculate Z value for each instrument per day\nstats['liquidity_imbalance'] = stats['std'] / stats['mean']\n# Display the calculated Z values\nresult=stats['liquidity_imbalance']\nresult.index.names = ['datetime','instrument']\n# result = result.swaplevel().sort_index()\nresult.to_hdf('result.h5', key='data')"
|
||||
},
|
||||
"liquidity_imbalance_2": {
|
||||
"description": "liquidity_imbalance=std(minute trading liquidity_imbalance)/mean(minute trading liquidity_imbalance).",
|
||||
"formulation": "liquidity_imbalance = frac{text{std}(text{minute trading liquidity_imbalance})}{text{mean}(text{minute liquidity_imbalance})}",
|
||||
"variables": {
|
||||
"std(minute liquidity_imbalance)": "Standard deviation of trading liquidity_imbalance for each minute of the trading day.",
|
||||
"mean(minute liquidity_imbalance)": "Mean of trading liquidity_imbalance for each minute of the trading day.",
|
||||
"liquidity_imbalance": "(bid_size-ask_size)/2*(bid_size+ask_size), we use something like bidV for the size"
|
||||
},
|
||||
"Category": "High-Frequency",
|
||||
"Difficulty": "Medium",
|
||||
"gt_code": "import pandas as pd\ndata_hf = pd.read_hdf('high_freq.h5')\nsample_df= data_hf.reset_index()\n# Convert 'datetime' column to datetime and extract date for grouping\nsample_df['date'] = sample_df['datetime'].dt.date\nsample_df['liquidity_imbalance']=(sample_df['bidV']-sample_df['askV'])/((sample_df['bidV']+sample_df['askV'])*2)\n# Group by instrument and date\ngrouped = sample_df.groupby(['date','instrument'])['liquidity_imbalance']\n# Calculate mean and standard deviation of the volume for each group\nstats = grouped.agg(['mean', 'std'])\n# Calculate Z value for each instrument per day\nstats['liquidity_imbalance'] = stats['std'] / stats['mean']\n# Display the calculated Z values\nresult=stats['liquidity_imbalance']\nresult.index.names = ['datetime','instrument']\n# result = result.swaplevel().sort_index()\nresult.to_hdf('result.h5', key='data')"
|
||||
},
|
||||
"liquidity_imbalance_3": {
|
||||
"description": "liquidity_imbalance=std(minute trading liquidity_imbalance)/mean(minute trading liquidity_imbalance).",
|
||||
"formulation": "liquidity_imbalance = frac{text{std}(text{minute trading liquidity_imbalance})}{text{mean}(text{minute liquidity_imbalance})}",
|
||||
"variables": {
|
||||
"std(minute liquidity_imbalance)": "Standard deviation of trading liquidity_imbalance for each minute of the trading day.",
|
||||
"mean(minute liquidity_imbalance)": "Mean of trading liquidity_imbalance for each minute of the trading day.",
|
||||
"liquidity_imbalance": "(bid_size-ask_size)/3*(bid_size+ask_size), we use something like bidV for the size"
|
||||
},
|
||||
"Category": "High-Frequency",
|
||||
"Difficulty": "Medium",
|
||||
"gt_code": "import pandas as pd\ndata_hf = pd.read_hdf('high_freq.h5')\nsample_df= data_hf.reset_index()\n# Convert 'datetime' column to datetime and extract date for grouping\nsample_df['date'] = sample_df['datetime'].dt.date\nsample_df['liquidity_imbalance']=(sample_df['bidV']-sample_df['askV'])/((sample_df['bidV']+sample_df['askV'])*3)\n# Group by instrument and date\ngrouped = sample_df.groupby(['date','instrument'])['liquidity_imbalance']\n# Calculate mean and standard deviation of the volume for each group\nstats = grouped.agg(['mean', 'std'])\n# Calculate Z value for each instrument per day\nstats['liquidity_imbalance'] = stats['std'] / stats['mean']\n# Display the calculated Z values\nresult=stats['liquidity_imbalance']\nresult.index.names = ['datetime','instrument']\n# result = result.swaplevel().sort_index()\nresult.to_hdf('result.h5', key='data')"
|
||||
},
|
||||
"micro_price": {
|
||||
"description": "micro_price=std(minute trading micro_price)/mean(minute trading micro_price).",
|
||||
"formulation": "micro_price = frac{text{std}(text{minute trading micro_price})}{text{mean}(text{minute micro_price})}",
|
||||
"variables": {
|
||||
"std(minute micro_price)": "Standard deviation of trading micro_price for each minute of the trading day.",
|
||||
"mean(minute micro_price)": "Mean of trading micro_price for each minute of the trading day.",
|
||||
"micro_price": "((df['bid_price'] * df['ask_size']) + (df['ask_price'] * df['bid_size'])) / (df['bid_size'] + df['ask_size'])"
|
||||
},
|
||||
"Category": "High-Frequency",
|
||||
"Difficulty": "Hard",
|
||||
"gt_code": "import pandas as pd\ndata_hf = pd.read_hdf('high_freq.h5')\nsample_df= data_hf.reset_index()\n# Convert 'datetime' column to datetime and extract date for grouping\nsample_df['date'] = sample_df['datetime'].dt.date\nsample_df['micro_price']=(sample_df['bid']*sample_df['askV']+sample_df['ask']*sample_df['bidV'])/(sample_df['bidV']+sample_df['askV'])\n# Group by instrument and date\ngrouped = sample_df.groupby(['date','instrument'])['micro_price']\n# Calculate mean and standard deviation of the volume for each group\nstats = grouped.agg(['mean', 'std'])\n# Calculate Z value for each instrument per day\nstats['micro_price'] = stats['std'] / stats['mean']\n# Display the calculated Z values\nresult=stats['micro_price']\nresult.index.names = ['datetime','instrument']\n# result = result.swaplevel().sort_index()\nresult.to_hdf('result.h5', key='data')"
|
||||
},
|
||||
"micro_price_2": {
|
||||
"description": "micro_price_2=std(minute trading micro_price)/mean(minute trading micro_price).",
|
||||
"formulation": "micro_price_2 = frac{text{std}(text{minute trading micro_price})}{text{mean}(text{minute micro_price})}",
|
||||
"variables": {
|
||||
"std(minute micro_price)": "Standard deviation of trading micro_price for each minute of the trading day.",
|
||||
"mean(minute micro_price)": "Mean of trading micro_price for each minute of the trading day.",
|
||||
"micro_price": "((df['bid_price'] * df['ask_size']) + (df['ask_price'] * df['bid_size'])) / 2*(df['bid_size'] + df['ask_size']), we use something like bidV for the size"
|
||||
},
|
||||
"Category": "High-Frequency",
|
||||
"Difficulty": "Hard",
|
||||
"gt_code": "import pandas as pd\ndata_hf = pd.read_hdf('high_freq.h5')\nsample_df= data_hf.reset_index()\n# Convert 'datetime' column to datetime and extract date for grouping\nsample_df['date'] = sample_df['datetime'].dt.date\nsample_df['micro_price']=(sample_df['bid']*sample_df['askV']+sample_df['ask']*sample_df['bidV'])/((sample_df['bidV']+sample_df['askV'])*2)\n# Group by instrument and date\ngrouped = sample_df.groupby(['date','instrument'])['micro_price']\n# Calculate mean and standard deviation of the volume for each group\nstats = grouped.agg(['mean', 'std'])\n# Calculate Z value for each instrument per day\nstats['micro_price'] = stats['std'] / stats['mean']\n# Display the calculated Z values\nresult=stats['micro_price']\nresult.index.names = ['datetime','instrument']\n# result = result.swaplevel().sort_index()\nresult.to_hdf('result.h5', key='data')"
|
||||
},
|
||||
"micro_price_3": {
|
||||
"description": "micro_price_3=std(minute trading micro_price)/mean(minute trading micro_price).",
|
||||
"formulation": "micro_price_3 = frac{text{std}(text{minute trading micro_price})}{text{mean}(text{minute micro_price})}",
|
||||
"variables": {
|
||||
"std(minute micro_price)": "Standard deviation of trading micro_price for each minute of the trading day.",
|
||||
"mean(minute micro_price)": "Mean of trading micro_price for each minute of the trading day.",
|
||||
"micro_price": "((df['bid_price'] * df['ask_size']) + (df['ask_price'] * df['bid_size'])) / 3*(df['bid_size'] + df['ask_size']), we use something like bidV for the size"
|
||||
},
|
||||
"Category": "High-Frequency",
|
||||
"Difficulty": "Hard",
|
||||
"gt_code": "import pandas as pd\ndata_hf = pd.read_hdf('high_freq.h5')\nsample_df= data_hf.reset_index()\n# Convert 'datetime' column to datetime and extract date for grouping\nsample_df['date'] = sample_df['datetime'].dt.date\nsample_df['micro_price']=(sample_df['bid']*sample_df['askV']+sample_df['ask']*sample_df['bidV'])/((sample_df['bidV']+sample_df['askV'])*3)\n# Group by instrument and date\ngrouped = sample_df.groupby(['date','instrument'])['micro_price']\n# Calculate mean and standard deviation of the volume for each group\nstats = grouped.agg(['mean', 'std'])\n# Calculate Z value for each instrument per day\nstats['micro_price'] = stats['std'] / stats['mean']\n# Display the calculated Z values\nresult=stats['micro_price']\nresult.index.names = ['datetime','instrument']\n# result = result.swaplevel().sort_index()\nresult.to_hdf('result.h5', key='data')"
|
||||
},
|
||||
"mid_price": {
|
||||
"description": "mid_price=std(minute trading mid_price)/mean(minute trading mid_price).",
|
||||
"formulation": "mid_price = frac{text{std}(text{minute trading mid price})}{text{mean}(text{minute mid price})}",
|
||||
"variables": {
|
||||
"std(minute mid_price)": "Standard deviation of trading mid_price for each minute of the trading day.",
|
||||
"mean(minute mid_price)": "Mean of trading mid_price for each minute of the trading day.",
|
||||
"mid_price": "The average of the bid and ask prices."
|
||||
},
|
||||
"Category": "High-Frequency",
|
||||
"Difficulty": "Easy",
|
||||
"gt_code": "import pandas as pd\ndata_hf = pd.read_hdf('high_freq.h5')\nsample_df= data_hf.reset_index()\n# Convert 'datetime' column to datetime and extract date for grouping\nsample_df['date'] = sample_df['datetime'].dt.date\nsample_df['mid_price']=(sample_df['bid']+sample_df['ask'])/2\n# Group by instrument and date\ngrouped = sample_df.groupby(['date','instrument'])['mid_price']\n# Calculate mean and standard deviation of the volume for each group\nstats = grouped.agg(['mean', 'std'])\nstats['mid_price'] = stats['std'] / stats['mean']\nresult=stats['mid_price']\nresult.index.names = ['datetime','instrument']\n# result = result.swaplevel().sort_index()\nresult.to_hdf('result.h5', key='data')"
|
||||
},
|
||||
"mid_price_2": {
|
||||
"description": "mid_price=std(minute trading mid_price)/mean(minute trading mid_price).",
|
||||
"formulation": "mid_price = frac{text{std}(text{minute trading mid price})}{text{mean}(text{minute mid price})}",
|
||||
"variables": {
|
||||
"std(minute mid_price)": "Standard deviation of trading mid_price for each minute of the trading day.",
|
||||
"mean(minute mid_price)": "Mean of trading mid_price for each minute of the trading day.",
|
||||
"mid_price_2": "the average of the bid and ask prices plus the the average of the bid and ask size (bidV and askV)."
|
||||
},
|
||||
"Category": "High-Frequency",
|
||||
"Difficulty": "Easy",
|
||||
"gt_code": "import pandas as pd\ndata_hf = pd.read_hdf('high_freq.h5')\nsample_df= data_hf.reset_index()\n# Convert 'datetime' column to datetime and extract date for grouping\nsample_df['date'] = sample_df['datetime'].dt.date\nsample_df['mid_price']=(sample_df['bid']+sample_df['ask'])/2+(sample_df['bidV']+sample_df['askV'])/2\n# Group by instrument and date\ngrouped = sample_df.groupby(['date','instrument'])['mid_price']\n# Calculate mean and standard deviation of the volume for each group\nstats = grouped.agg(['mean', 'std'])\nstats['mid_price'] = stats['std'] / stats['mean']\nresult=stats['mid_price']\nresult.index.names = ['datetime','instrument']\n# result = result.swaplevel().sort_index()\nresult.to_hdf('result.h5', key='data')"
|
||||
},
|
||||
"mid_price_3": {
|
||||
"description": "mid_price=std(minute trading mid_price)/mean(minute trading mid_price).",
|
||||
"formulation": "mid_price = frac{text{std}(text{minute trading mid price})}{text{mean}(text{minute mid price})}",
|
||||
"variables": {
|
||||
"std(minute mid_price)": "Standard deviation of trading mid_price for each minute of the trading day.",
|
||||
"mean(minute mid_price)": "Mean of trading mid_price for each minute of the trading day.",
|
||||
"mid_price_3": "The coefficient of variation (CV) of the mid-price for each minute of the trading day, calculated as the standard deviation of the mid-price divided by the mean mid-price."
|
||||
},
|
||||
"Category": "High-Frequency",
|
||||
"Difficulty": "Easy",
|
||||
"gt_code": "import pandas as pd\ndata_hf = pd.read_hdf('high_freq.h5')\nsample_df= data_hf.reset_index()\n# Convert 'datetime' column to datetime and extract date for grouping\nsample_df['date'] = sample_df['datetime'].dt.date\nsample_df['mid_price']=(sample_df['bid']+sample_df['ask'])/3\n# Group by instrument and date\ngrouped = sample_df.groupby(['date','instrument'])['mid_price']\n# Calculate mean and standard deviation of the volume for each group\nstats = grouped.agg(['mean', 'std'])\nstats['mid_price'] = stats['std'] / stats['mean']\nresult=stats['mid_price']\nresult.index.names = ['datetime','instrument']\n# result = result.swaplevel().sort_index()\nresult.to_hdf('result.h5', key='data')"
|
||||
},
|
||||
"PB_ROE": {
|
||||
"description": "Constructed using the ranking difference between PB and ROE, with regression versions of PB and ROE replacing original PB and ROE to obtain reconstructed factor values.",
|
||||
"formulation": "text{rank}(PB_t) - rank(ROE_t)",
|
||||
"variables": {
|
||||
"text{rank}(PB_t)": "Ranking of regression version PB on cross-section at time t.",
|
||||
"text{rank}(ROE_t)": "Ranking of regression version single-quarter ROE on cross-section at time t."
|
||||
},
|
||||
"Category": "Fundamentals",
|
||||
"Difficulty": "Easy",
|
||||
"gt_code": "import pandas as pd\ndata_f = pd.read_hdf('daily_f.h5')\ndata = data_f.reset_index()\n# Calculate the rank of PB and ROE\ndata['PB_rank'] = data.groupby('datetime')['B/P'].rank()\ndata['ROE_rank'] = data.groupby('datetime')['ROE'].rank()\n# Calculate the difference between the ranks\ndata['PB_ROE'] = data['PB_rank'] - data['ROE_rank']\n# set the datetime and instrument as index and drop the original index\nresult=pd.DataFrame(data['PB_ROE']).set_index(data_f.index)\n# transfer the result to series\nresult=result['PB_ROE']\nresult.to_hdf('result.h5', key='data')"
|
||||
},
|
||||
"PB_ROE_2": {
|
||||
"description": "Constructed using the ranking difference between PB/2 and ROE, with regression versions of PB and ROE replacing original PB and ROE to obtain reconstructed factor values.",
|
||||
"formulation": "text{rank}(PB_t)/2 - rank(ROE_t)",
|
||||
"variables": {
|
||||
"text{rank}(PB_t)": "Ranking of regression version PB on cross-section at time t.",
|
||||
"text{rank}(ROE_t)": "Ranking of regression version single-quarter ROE on cross-section at time t."
|
||||
},
|
||||
"Category": "Fundamentals",
|
||||
"Difficulty": "Easy",
|
||||
"gt_code": "import pandas as pd\ndata_f = pd.read_hdf('daily_f.h5')\ndata = data_f.reset_index()\n# Calculate the rank of PB and ROE\ndata['PB_rank'] = data.groupby('datetime')['B/P'].rank()\ndata['ROE_rank'] = data.groupby('datetime')['ROE'].rank()\n# Calculate the difference between the ranks\ndata['PB_ROE'] = data['PB_rank']/2 - data['ROE_rank']\n# set the datetime and instrument as index and drop the original index\nresult=pd.DataFrame(data['PB_ROE']).set_index(data_f.index)\n# transfer the result to series\nresult=result['PB_ROE']\nresult.to_hdf('result.h5', key='data')"
|
||||
},
|
||||
"PB_ROE_3": {
|
||||
"description": "Constructed using the ranking difference between PB/3 and ROE, with regression versions of PB and ROE replacing original PB and ROE to obtain reconstructed factor values.",
|
||||
"formulation": "text{rank}(PB_t)/3 - rank(ROE_t)",
|
||||
"variables": {
|
||||
"text{rank}(PB_t)": "Ranking of regression version PB on cross-section at time t.",
|
||||
"text{rank}(ROE_t)": "Ranking of regression version single-quarter ROE on cross-section at time t."
|
||||
},
|
||||
"Category": "Fundamentals",
|
||||
"Difficulty": "Easy",
|
||||
"gt_code": "import pandas as pd\ndata_f = pd.read_hdf('daily_f.h5')\ndata = data_f.reset_index()\n# Calculate the rank of PB and ROE\ndata['PB_rank'] = data.groupby('datetime')['B/P'].rank()\ndata['ROE_rank'] = data.groupby('datetime')['ROE'].rank()\n# Calculate the difference between the ranks\ndata['PB_ROE'] = data['PB_rank']/3 - data['ROE_rank']\n# set the datetime and instrument as index and drop the original index\nresult=pd.DataFrame(data['PB_ROE']).set_index(data_f.index)\n# transfer the result to series\nresult=result['PB_ROE']\nresult.to_hdf('result.h5', key='data')"
|
||||
},
|
||||
"PB_ROE_movement": {
|
||||
"description": "PB_ROE_movement=five day PB_ROE movement indicator(-1 and 1 or 0).",
|
||||
"formulation": "PB_ROE_movement = 5_day_movement(PB_ROE), PB_ROE = text{rank}(PB_t) - rank(ROE_t)",
|
||||
"variables": {
|
||||
"PB_ROE": "the ranking difference between PB and ROE.",
|
||||
"5_day_PB_ROE_movement": "1 if PB_ROE is higher than the PB_ROE 5 days ago, -1 if PB_ROE is lower than the PB_ROE 5 days ago, 0 if PB_ROE is the same as the PB_ROE 5 days ago.",
|
||||
"text{rank}(PB_t)": "Ranking of regression version PB on cross-section at time t.",
|
||||
"text{rank}(ROE_t)": "Ranking of regression version single-quarter ROE on cross-section at time t."
|
||||
},
|
||||
"Category": "Fundamentals",
|
||||
"Difficulty": "Hard",
|
||||
"gt_code": "import pandas as pd\ndata_f = pd.read_hdf('daily_f.h5')\nsample_df = data_f.reset_index()\n# Calculate the rank of PB and ROE\nsample_df['PB_rank'] = sample_df.groupby('datetime')['B/P'].rank()\nsample_df['ROE_rank'] = sample_df.groupby('datetime')['ROE'].rank()\nsample_df['PB_ROE'] = sample_df['PB_rank'] - sample_df['ROE_rank']\n# Group by instrument and date\nsample_df['PB_ROE_movement'] = sample_df['PB_ROE'].diff(periods=5).apply(lambda x: 1 if x > 0 else (-1 if x < 0 else 0))\n#calculate the mid_price_movement ratio for each day\n# set the datetime and instrument as index and drop the original index\nresult=pd.DataFrame(sample_df['PB_ROE_movement']).set_index(data_f.index)\n# transfer the result to series\nresult=result['PB_ROE_movement']\nresult.to_hdf('result.h5', key='data')"
|
||||
},
|
||||
"PB_ROE_movement_10": {
|
||||
"description": "PB_ROE_movement=10 days PB_ROE movement indicator(-1 and 1 or 0).",
|
||||
"formulation": "PB_ROE_movement = 10_day_movement(PB_ROE), PB_ROE = text{rank}(PB_t) - rank(ROE_t)",
|
||||
"variables": {
|
||||
"PB_ROE": "the ranking difference between PB and ROE.",
|
||||
"10_day_PB_ROE_movement": "1 if PB_ROE is higher than the PB_ROE 10 days ago, -1 if PB_ROE is lower than the PB_ROE 10 days ago, 0 if PB_ROE is the same as the PB_ROE 10 days ago.",
|
||||
"text{rank}(PB_t)": "Ranking of regression version PB on cross-section at time t.",
|
||||
"text{rank}(ROE_t)": "Ranking of regression version single-quarter ROE on cross-section at time t."
|
||||
},
|
||||
"Category": "Fundamentals",
|
||||
"Difficulty": "Hard",
|
||||
"gt_code": "import pandas as pd\ndata_f = pd.read_hdf('daily_f.h5')\nsample_df = data_f.reset_index()\n# Calculate the rank of PB and ROE\nsample_df['PB_rank'] = sample_df.groupby('datetime')['B/P'].rank()\nsample_df['ROE_rank'] = sample_df.groupby('datetime')['ROE'].rank()\nsample_df['PB_ROE'] = sample_df['PB_rank'] - sample_df['ROE_rank']\n# Group by instrument and date\nsample_df['PB_ROE_movement'] = sample_df['PB_ROE'].diff(periods=10).apply(lambda x: 1 if x > 0 else (-1 if x < 0 else 0))\n#calculate the mid_price_movement ratio for each day\n# set the datetime and instrument as index and drop the original index\nresult=pd.DataFrame(sample_df['PB_ROE_movement']).set_index(data_f.index)\n# transfer the result to series\nresult=result['PB_ROE_movement']\nresult.to_hdf('result.h5', key='data')"
|
||||
},
|
||||
"PB_ROE_movement_20": {
|
||||
"description": "PB_ROE_movement=20 days PB_ROE movement indicator(-1 and 1 or 0).",
|
||||
"formulation": "PB_ROE_movement = 20_day_movement(PB_ROE), PB_ROE = text{rank}(PB_t) - rank(ROE_t)",
|
||||
"variables": {
|
||||
"PB_ROE": "the ranking difference between PB and ROE.",
|
||||
"20_day_PB_ROE_movement": "1 if PB_ROE is higher than the PB_ROE 20 days ago, -1 if PB_ROE is lower than the PB_ROE 20 days ago, 0 if PB_ROE is the same as the PB_ROE 20 days ago.",
|
||||
"text{rank}(PB_t)": "Ranking of regression version PB on cross-section at time t.",
|
||||
"text{rank}(ROE_t)": "Ranking of regression version single-quarter ROE on cross-section at time t."
|
||||
},
|
||||
"Category": "Fundamentals",
|
||||
"Difficulty": "Hard",
|
||||
"gt_code": "import pandas as pd\ndata_f = pd.read_hdf('daily_f.h5')\nsample_df = data_f.reset_index()\n# Calculate the rank of PB and ROE\nsample_df['PB_rank'] = sample_df.groupby('datetime')['B/P'].rank()\nsample_df['ROE_rank'] = sample_df.groupby('datetime')['ROE'].rank()\nsample_df['PB_ROE'] = sample_df['PB_rank'] - sample_df['ROE_rank']\n# Group by instrument and date\nsample_df['PB_ROE_movement'] = sample_df['PB_ROE'].diff(periods=20).apply(lambda x: 1 if x > 0 else (-1 if x < 0 else 0))\n#calculate the mid_price_movement ratio for each day\n# set the datetime and instrument as index and drop the original index\nresult=pd.DataFrame(sample_df['PB_ROE_movement']).set_index(data_f.index)\n# transfer the result to series\nresult=result['PB_ROE_movement']\nresult.to_hdf('result.h5', key='data')"
|
||||
},
|
||||
"ROE_movement": {
|
||||
"description": "ROE_movement=five day ROE movement indicator(-1 and 1 or 0).",
|
||||
"formulation": "ROE_movement = 5_day_movement(ROE)",
|
||||
"variables": {
|
||||
"ROE": "ROE in fundamental statistics.",
|
||||
"5_day_ROE_movement": "1 if ROE is higher than the ROE 5 days ago, -1 if ROE is lower than the ROE 5 days ago, 0 if ROE is the same as the ROE 5 days ago."
|
||||
},
|
||||
"Category": "Fundamentals",
|
||||
"Difficulty": "Medium",
|
||||
"gt_code": "import pandas as pd\ndata_f = pd.read_hdf('daily_f.h5')\nsample_df = data_f.reset_index()\n# Group by instrument and date\nsample_df['ROE_movement'] = sample_df['ROE'].diff(periods=5).apply(lambda x: 1 if x > 0 else (-1 if x < 0 else 0))\n#calculate the mid_price_movement ratio for each day\n# set the datetime and instrument as index and drop the original index\nresult=pd.DataFrame(sample_df['ROE_movement']).set_index(data_f.index)\n# transfer the result to series\nresult=result['ROE_movement']\nresult.to_hdf('result.h5', key='data')"
|
||||
},
|
||||
"ROE_movement_10": {
|
||||
"description": "ROE_movement_10=ten day ROE movement indicator(-1 and 1 or 0).",
|
||||
"formulation": "ROE_movement = 10_day_movement(ROE)",
|
||||
"variables": {
|
||||
"ROE": "ROE in fundamental statistics.",
|
||||
"10_day_ROE_movement": "1 if ROE is higher than the ROE 10 days ago, -1 if ROE is lower than the ROE 10 days ago, 0 if ROE is the same as the ROE 10 days ago."
|
||||
},
|
||||
"Category": "Fundamentals",
|
||||
"Difficulty": "Medium",
|
||||
"gt_code": "import pandas as pd\ndata_f = pd.read_hdf('daily_f.h5')\nsample_df = data_f.reset_index()\n# Group by instrument and date\nsample_df['ROE_movement'] = sample_df['ROE'].diff(periods=10).apply(lambda x: 1 if x > 0 else (-1 if x < 0 else 0))\n#calculate the mid_price_movement ratio for each day\n# set the datetime and instrument as index and drop the original index\nresult=pd.DataFrame(sample_df['ROE_movement']).set_index(data_f.index)\n# transfer the result to series\nresult=result['ROE_movement']\nresult.to_hdf('result.h5', key='data')"
|
||||
},
|
||||
"ROE_movement_20": {
|
||||
"description": "ROE_movement_20=20 day ROE movement indicator(-1 and 1 or 0).",
|
||||
"formulation": "ROE_movement_20 = 20_day_movement(ROE)",
|
||||
"variables": {
|
||||
"ROE": "ROE in fundamental statistics.",
|
||||
"20_day_ROE_movement": "1 if ROE is higher than the ROE 20 days ago, -1 if ROE is lower than the ROE 20 days ago, 0 if ROE is the same as the ROE 20 days ago."
|
||||
},
|
||||
"Category": "Fundamentals",
|
||||
"Difficulty": "Medium",
|
||||
"gt_code": "import pandas as pd\ndata_f = pd.read_hdf('daily_f.h5')\nsample_df = data_f.reset_index()\n# Group by instrument and date\nsample_df['ROE_movement'] = sample_df['ROE'].diff(periods=20).apply(lambda x: 1 if x > 0 else (-1 if x < 0 else 0))\n#calculate the mid_price_movement ratio for each day\n# set the datetime and instrument as index and drop the original index\nresult=pd.DataFrame(sample_df['ROE_movement']).set_index(data_f.index)\n# transfer the result to series\nresult=result['ROE_movement']\nresult.to_hdf('result.h5', key='data')"
|
||||
},
|
||||
"alpha_pv_diff": {
|
||||
"description": "alpha_pv_diff is defined as the ratio of the difference between close prices 10 days change and open prices 10 days change to the sum of the highest minus lowest prices plus a small constant.",
|
||||
"formulation": "frac{(text{close_diff10} - text{open_diff10})}{(text{high} - text{low} + 0.001)}",
|
||||
"variables": {
|
||||
"close": "Closing price of the stock",
|
||||
"open": "Opening price of the stock",
|
||||
"high": "Highest price of the stock during the day",
|
||||
"low": "Lowest price of the stock during the day"
|
||||
},
|
||||
"Category": "Volume&Price",
|
||||
"Difficulty": "Medium",
|
||||
"gt_code": "import pandas as pd\ndata_pv = pd.read_hdf('daily_pv.h5')\nnew_df= data_pv.reset_index()\n# Calculate Alpha101\nnew_df['result'] = (new_df['$close'].diff(10) - new_df['$open'].diff(10)) / (new_df['$high'] - new_df['$low'] + 0.001)\n# keep the index of the original dataframe\nresult=pd.DataFrame(new_df['result']).set_index(data_pv.index)\n# transfer the result to series\nresult=result['result']\nresult.to_hdf('result.h5', key='data')"
|
||||
},
|
||||
"alpha_pv_diff_15": {
|
||||
"description": "alpha_pv_diff is defined as the ratio of the difference between close prices 15 days change and open prices 15 days change to the sum of the highest minus lowest prices plus a small constant.",
|
||||
"formulation": "frac{(text{close_diff15} - text{open_diff15})}{(text{high} - text{low} + 0.001)}",
|
||||
"variables": {
|
||||
"close": "Closing price of the stock",
|
||||
"open": "Opening price of the stock",
|
||||
"high": "Highest price of the stock during the day",
|
||||
"low": "Lowest price of the stock during the day"
|
||||
},
|
||||
"Category": "Volume&Price",
|
||||
"Difficulty": "Medium",
|
||||
"gt_code": "import pandas as pd\ndata_pv = pd.read_hdf('daily_pv.h5')\nnew_df= data_pv.reset_index()\n# Calculate Alpha101\nnew_df['result'] = (new_df['$close'].diff(15) - new_df['$open'].diff(15)) / (new_df['$high'] - new_df['$low'] + 0.001)\n# keep the index of the original dataframe\nresult=pd.DataFrame(new_df['result']).set_index(data_pv.index)\n# transfer the result to series\nresult=result['result']\nresult.to_hdf('result.h5', key='data')"
|
||||
},
|
||||
"alpha_pv_diff_20": {
|
||||
"description": "alpha_pv_diff is defined as the ratio of the difference between close prices 20 days change and open prices 20 days change to the sum of the highest minus lowest prices plus a small constant.",
|
||||
"formulation": "frac{(text{close_diff20} - text{open_diff20})}{(text{high} - text{low} + 0.001)}",
|
||||
"variables": {
|
||||
"close": "Closing price of the stock",
|
||||
"open": "Opening price of the stock",
|
||||
"high": "Highest price of the stock during the day",
|
||||
"low": "Lowest price of the stock during the day"
|
||||
},
|
||||
"Category": "Volume&Price",
|
||||
"Difficulty": "Medium",
|
||||
"gt_code": "import pandas as pd\ndata_pv = pd.read_hdf('daily_pv.h5')\nnew_df= data_pv.reset_index()\n# Calculate Alpha101\nnew_df['result'] = (new_df['$close'].diff(20) - new_df['$open'].diff(20)) / (new_df['$high'] - new_df['$low'] + 0.001)\n# keep the index of the original dataframe\nresult=pd.DataFrame(new_df['result']).set_index(data_pv.index)\n# transfer the result to series\nresult=result['result']\nresult.to_hdf('result.h5', key='data')"
|
||||
},
|
||||
"alpha_pv_diff_pct": {
|
||||
"description": "alpha_pv is defined as the ratio of the difference between close prices 10 days change and open prices 10 days change to the sum of the highest prices 10 days change ratio minus lowest prices 10 days change ratio plus a small constant.",
|
||||
"formulation": "frac{(text{close_diff10} - text{open_diff10})}{(text{high_pct10} - text{low_pct10} + 0.001)}",
|
||||
"variables": {
|
||||
"close": "Closing price of the stock",
|
||||
"open": "Opening price of the stock",
|
||||
"high": "Highest price of the stock during the day",
|
||||
"low": "Lowest price of the stock during the day"
|
||||
},
|
||||
"Category": "Volume&Price",
|
||||
"Difficulty": "Hard",
|
||||
"gt_code": "import pandas as pd\ndata_pv = pd.read_hdf('daily_pv.h5')\nnew_df= data_pv.reset_index()\n# Calculate Alpha101\nnew_df['result'] = (new_df['$close'].diff(10) - new_df['$open'].diff(10)) / (new_df['$high'].pct_change(10) - new_df['$low'].pct_change(10) + 0.001)\n# keep the index of the original dataframe\nresult=pd.DataFrame(new_df['result']).set_index(data_pv.index)\n# transfer the result to series\nresult=result['result']\nresult.to_hdf('result.h5', key='data')"
|
||||
},
|
||||
"alpha_pv_diff_pct_15": {
|
||||
"description": "alpha_pv is defined as the ratio of the difference between close prices 15 days change and open prices 15 days change to the sum of the highest prices 10 days change ratio minus lowest prices 10 days change ratio plus a small constant.",
|
||||
"formulation": "frac{(text{close_diff15} - text{open_diff15})}{(text{high_pct10} - text{low_pct10} + 0.001)}",
|
||||
"variables": {
|
||||
"close": "Closing price of the stock",
|
||||
"open": "Opening price of the stock",
|
||||
"high": "Highest price of the stock during the day",
|
||||
"low": "Lowest price of the stock during the day"
|
||||
},
|
||||
"Category": "Volume&Price",
|
||||
"Difficulty": "Hard",
|
||||
"gt_code": "import pandas as pd\ndata_pv = pd.read_hdf('daily_pv.h5')\nnew_df= data_pv.reset_index()\n# Calculate Alpha101\nnew_df['result'] = (new_df['$close'].diff(15) - new_df['$open'].diff(15)) / (new_df['$high'].pct_change(10) - new_df['$low'].pct_change(10) + 0.001)\n# keep the index of the original dataframe\nresult=pd.DataFrame(new_df['result']).set_index(data_pv.index)\n# transfer the result to series\nresult=result['result']\nresult.to_hdf('result.h5', key='data')"
|
||||
},
|
||||
"alpha_pv_diff_pct_20": {
|
||||
"description": "alpha_pv is defined as the ratio of the difference between close prices 20 days change and open prices 20 days change to the sum of the highest prices 10 days change ratio minus lowest prices 10 days change ratio plus a small constant.",
|
||||
"formulation": "frac{(text{close_diff20} - text{open_diff20})}{(text{high_pct10} - text{low_pct10} + 0.001)}",
|
||||
"variables": {
|
||||
"close": "Closing price of the stock",
|
||||
"open": "Opening price of the stock",
|
||||
"high": "Highest price of the stock during the day",
|
||||
"low": "Lowest price of the stock during the day"
|
||||
},
|
||||
"Category": "Volume&Price",
|
||||
"Difficulty": "Hard",
|
||||
"gt_code": "import pandas as pd\ndata_pv = pd.read_hdf('daily_pv.h5')\nnew_df= data_pv.reset_index()\n# Calculate Alpha101\nnew_df['result'] = (new_df['$close'].diff(20) - new_df['$open'].diff(20)) / (new_df['$high'].pct_change(10) - new_df['$low'].pct_change(10) + 0.001)\n# keep the index of the original dataframe\nresult=pd.DataFrame(new_df['result']).set_index(data_pv.index)\n# transfer the result to series\nresult=result['result']\nresult.to_hdf('result.h5', key='data')"
|
||||
},
|
||||
"alpha053": {
|
||||
"description": "Reversal class factor, negative delta of a ratio involving close, low, and high prices over 9 days.",
|
||||
"formulation": "-1 times Deltaleft(frac{(text{close} - text{low}) - (text{high} - text{close})}{text{close} - text{low}}, 9right)",
|
||||
"variables": {
|
||||
"Delta(x, d)": "Change in 'x' over 'd' days.",
|
||||
"text{close}": "Closing price of the stock.",
|
||||
"text{low}": "Lowest price of the stock for the day.",
|
||||
"text{high}": "Highest price of the stock for the day."
|
||||
},
|
||||
"Category": "Volume&Price",
|
||||
"Difficulty": "Easy",
|
||||
"gt_code": "import pandas as pd\ndata_pv = pd.read_hdf('daily_pv.h5')\nnew_df= data_pv.reset_index()\n# Calculate Alpha053\nnew_df['ratio'] = (new_df['$close'] - new_df['$low'] - (new_df['$high'] - new_df['$close'])) / (new_df['$close'] - new_df['$low'])\n# the change of ratio in new_df over the 9 days\nnew_df['result']=-new_df['ratio'].diff(9)\n# transfer the result to series\nresult=pd.DataFrame(new_df['result']).set_index(data_pv.index)\nresult=result['result']\nresult.to_hdf('result.h5', key='data')"
|
||||
},
|
||||
"alpha053_5": {
|
||||
"description": "Reversal class factor, negative delta of a ratio involving close, low, and high prices over 5 days.",
|
||||
"formulation": "-1 times Deltaleft(frac{(text{close} - text{low}) - (text{high} - text{close})}{text{close} - text{low}}, 5right)",
|
||||
"variables": {
|
||||
"Delta(x, d)": "Change in 'x' over 'd' days.",
|
||||
"text{close}": "Closing price of the stock.",
|
||||
"text{low}": "Lowest price of the stock for the day.",
|
||||
"text{high}": "Highest price of the stock for the day."
|
||||
},
|
||||
"Category": "Volume&Price",
|
||||
"Difficulty": "Easy",
|
||||
"gt_code": "import pandas as pd\ndata_pv = pd.read_hdf('daily_pv.h5')\nnew_df= data_pv.reset_index()\n# Calculate Alpha053\nnew_df['ratio'] = (new_df['$close'] - new_df['$low'] - (new_df['$high'] - new_df['$close'])) / (new_df['$close'] - new_df['$low'])\n# the change of ratio in new_df over the 5 days\nnew_df['result']=-new_df['ratio'].diff(5)\n# transfer the result to series\nresult=pd.DataFrame(new_df['result']).set_index(data_pv.index)\nresult=result['result']\nresult.to_hdf('result.h5', key='data')"
|
||||
}
|
||||
}
|
||||
Vendored
BIN
Binary file not shown.
|
After Width: | Height: | Size: 61 KiB |
Vendored
BIN
Binary file not shown.
|
After Width: | Height: | Size: 567 KiB |
Vendored
BIN
Binary file not shown.
|
After Width: | Height: | Size: 3.8 KiB |
Vendored
BIN
Binary file not shown.
|
After Width: | Height: | Size: 88 KiB |
Vendored
BIN
Binary file not shown.
|
After Width: | Height: | Size: 507 KiB |
Vendored
BIN
Binary file not shown.
|
After Width: | Height: | Size: 115 KiB |
Vendored
BIN
Binary file not shown.
|
After Width: | Height: | Size: 303 KiB |
@@ -0,0 +1,15 @@
|
||||
=============
|
||||
API Reference
|
||||
=============
|
||||
|
||||
Here you can find all ``RDAgent``'s interfaces.
|
||||
|
||||
|
||||
RD Loop
|
||||
=======
|
||||
|
||||
Research
|
||||
--------
|
||||
|
||||
.. automodule:: rdagent.core.proposal
|
||||
:members:
|
||||
@@ -0,0 +1,4 @@
|
||||
# Changelog
|
||||
|
||||
## [Unreleased]
|
||||
<!-- insertion marker -->
|
||||
@@ -0,0 +1,72 @@
|
||||
# Configuration file for the Sphinx documentation builder.
|
||||
#
|
||||
# For the full list of built-in configuration values, see the documentation:
|
||||
# https://www.sphinx-doc.org/en/master/usage/configuration.html
|
||||
|
||||
# -- Project information -----------------------------------------------------
|
||||
# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
|
||||
|
||||
import subprocess
|
||||
|
||||
latest_tag = subprocess.check_output(["git", "describe", "--tags", "--abbrev=0"], text=True).strip()
|
||||
|
||||
project = "RDAgent"
|
||||
copyright = "2024, Microsoft"
|
||||
author = "Microsoft"
|
||||
|
||||
# -- General configuration ---------------------------------------------------
|
||||
# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
|
||||
|
||||
extensions = ["sphinx.ext.autodoc", "sphinxcontrib.autodoc_pydantic"]
|
||||
|
||||
autodoc_member_order = "bysource"
|
||||
|
||||
# The suffix of source filenames.
|
||||
source_suffix = {".rst": "restructuredtext"}
|
||||
|
||||
# The encoding of source files.
|
||||
source_encoding = "utf-8"
|
||||
|
||||
# The main toctree document.
|
||||
master_doc = "index"
|
||||
|
||||
# The version info for the project you're documenting, acts as replacement for
|
||||
# |version| and |release|, also used in various other places throughout the
|
||||
# built documents.
|
||||
#
|
||||
# The short X.Y version.
|
||||
version = latest_tag
|
||||
release = latest_tag
|
||||
|
||||
# The language for content autogenerated by Sphinx. Refer to documentation for
|
||||
# a list of supported languages.
|
||||
language = "en"
|
||||
|
||||
# List of patterns, relative to source directory, that match files and
|
||||
# directories to ignore when looking for source files.
|
||||
exclude_patterns = ["build"]
|
||||
|
||||
# -- Options for HTML output -------------------------------------------------
|
||||
# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
|
||||
# The name of the Pygments (syntax highlighting) style to use.
|
||||
pygments_style = "sphinx"
|
||||
|
||||
try:
|
||||
import furo
|
||||
|
||||
html_theme = "furo"
|
||||
html_theme_options = {
|
||||
"navigation_with_keys": True,
|
||||
}
|
||||
except ImportError:
|
||||
html_theme = "default"
|
||||
|
||||
html_logo = "_static/logo.png"
|
||||
html_static_path = ["_static"]
|
||||
html_favicon = "_static/favicon.ico"
|
||||
|
||||
html_theme_options = {
|
||||
"source_repository": "https://github.com/microsoft/RD-Agent",
|
||||
"source_branch": "main",
|
||||
"source_directory": "docs/",
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
=========================
|
||||
For Development
|
||||
=========================
|
||||
|
||||
If you want to try the latest version or contribute to RD-Agent. You can install it from the source and follow the commands in this page.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
git clone https://github.com/microsoft/RD-Agent
|
||||
|
||||
|
||||
🔧Prepare for development
|
||||
=========================
|
||||
|
||||
- Set up the development environment.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
make dev
|
||||
|
||||
- Run linting and checking.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
make lint
|
||||
|
||||
|
||||
- Some linting issues can be fixed automatically. We have added a command in the Makefile for easy use.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
make auto-lint
|
||||
|
||||
|
||||
|
||||
Code Structure
|
||||
=========================
|
||||
|
||||
.. code-block:: text
|
||||
|
||||
📂 src
|
||||
➥ 📂 <project name>: avoid namespace conflict
|
||||
➥ 📁 core
|
||||
➥ 📁 components/A
|
||||
➥ 📁 components/B
|
||||
➥ 📁 components/C
|
||||
➥ 📁 scenarios/X
|
||||
➥ 📁 scenarios/Y
|
||||
➥ 📂 app
|
||||
➥ 📁 scripts
|
||||
|
||||
.. list-table::
|
||||
:header-rows: 1
|
||||
|
||||
* - Folder Name
|
||||
- Description
|
||||
* - 📁 core
|
||||
- The core framework of the system. All classes should be abstract and usually can't be used directly.
|
||||
* - 📁 component/A
|
||||
- Useful components that can be used by others (e.g., scenarios). Many subclasses of core classes are located here.
|
||||
* - 📁 scenarios/X
|
||||
- Concrete features for specific scenarios (usually built based on components or core). These modules are often unreusable across scenarios.
|
||||
* - 📁 app
|
||||
- Applications for specific scenarios (usually built based on components or scenarios). Removing any of them does not affect the system's completeness or other scenarios.
|
||||
* - 📁 scripts
|
||||
- Quick and dirty things. These are candidates for core, components, scenarios, and apps.
|
||||
|
||||
|
||||
|
||||
Conventions
|
||||
===========
|
||||
|
||||
|
||||
File Naming Convention
|
||||
----------------------
|
||||
|
||||
.. list-table::
|
||||
:header-rows: 1
|
||||
|
||||
* - Name
|
||||
- Description
|
||||
* - `conf.py`
|
||||
- The configuration for the module, app, and project.
|
||||
|
||||
.. <!-- TODO: renaming files -->
|
||||
@@ -0,0 +1,34 @@
|
||||
.. RDAgent documentation master file, created by
|
||||
sphinx-quickstart on Mon Jul 15 04:27:50 2024.
|
||||
You can adapt this file completely to your liking, but it should at least
|
||||
contain the root `toctree` directive.
|
||||
|
||||
Welcome to RDAgent's documentation!
|
||||
===================================
|
||||
|
||||
.. image:: _static/logo.png
|
||||
:alt: RD-Agent Logo
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 3
|
||||
:caption: Doctree:
|
||||
|
||||
introduction
|
||||
installation_and_configuration
|
||||
scens/catalog
|
||||
project_framework_introduction
|
||||
ui
|
||||
research/catalog
|
||||
development
|
||||
api_reference
|
||||
policy
|
||||
|
||||
GitHub <https://github.com/microsoft/RD-Agent>
|
||||
|
||||
|
||||
Indices and tables
|
||||
==================
|
||||
|
||||
* :ref:`genindex`
|
||||
* :ref:`modindex`
|
||||
* :ref:`search`
|
||||
@@ -0,0 +1,446 @@
|
||||
==============================
|
||||
Installation and Configuration
|
||||
==============================
|
||||
|
||||
Installation
|
||||
============
|
||||
|
||||
**Install RDAgent**: For different scenarios
|
||||
|
||||
- for purely users: please use ``pip install rdagent`` to install RDAgent
|
||||
- for dev users: `See development <development.html>`_
|
||||
|
||||
**Install Docker**: RDAgent is designed for research and development, acting like a human researcher and developer. It can write and run code in various environments, primarily using Docker for code execution. This keeps the remaining dependencies simple. Users must ensure Docker is installed before attempting most scenarios. Please refer to the `official 🐳Docker page <https://docs.docker.com/engine/install/>`_ for installation instructions.
|
||||
Ensure the current user can run Docker commands **without using sudo**. You can verify this by executing `docker run hello-world`.
|
||||
|
||||
LiteLLM Backend Configuration (Default)
|
||||
=======================================
|
||||
|
||||
.. note::
|
||||
🔥 **Attention**: We now provide experimental support for **DeepSeek** models! You can use DeepSeek's official API for cost-effective and high-performance inference. See the configuration example below for DeepSeek setup.
|
||||
|
||||
Option 1: Unified API base for both models
|
||||
------------------------------------------
|
||||
|
||||
.. code-block:: Properties
|
||||
|
||||
# Set to any model supported by LiteLLM.
|
||||
CHAT_MODEL=gpt-4o
|
||||
EMBEDDING_MODEL=text-embedding-3-small
|
||||
# Configure unified API base
|
||||
# The backend api_key fully follows the convention of litellm.
|
||||
OPENAI_API_BASE=<your_unified_api_base>
|
||||
OPENAI_API_KEY=<replace_with_your_openai_api_key>
|
||||
|
||||
Option 2: Separate API bases for Chat and Embedding models
|
||||
----------------------------------------------------------
|
||||
|
||||
.. code-block:: Properties
|
||||
|
||||
# Set to any model supported by LiteLLM.
|
||||
|
||||
# CHAT MODEL:
|
||||
CHAT_MODEL=gpt-4o
|
||||
OPENAI_API_BASE=<your_chat_api_base>
|
||||
OPENAI_API_KEY=<replace_with_your_openai_api_key>
|
||||
|
||||
# EMBEDDING MODEL:
|
||||
# TAKE siliconflow as an example, you can use other providers.
|
||||
# Note: embedding requires litellm_proxy prefix
|
||||
EMBEDDING_MODEL=litellm_proxy/BAAI/bge-large-en-v1.5
|
||||
LITELLM_PROXY_API_KEY=<replace_with_your_siliconflow_api_key>
|
||||
LITELLM_PROXY_API_BASE=https://api.siliconflow.cn/v1
|
||||
|
||||
Configuration Example: DeepSeek Setup
|
||||
-------------------------------------
|
||||
|
||||
Many users encounter configuration errors when setting up DeepSeek. Here's a complete working example:
|
||||
|
||||
.. code-block:: Properties
|
||||
|
||||
# CHAT MODEL: Using DeepSeek Official API
|
||||
CHAT_MODEL=deepseek/deepseek-chat
|
||||
DEEPSEEK_API_KEY=<replace_with_your_deepseek_api_key>
|
||||
|
||||
# EMBEDDING MODEL: Using SiliconFlow for embedding since DeepSeek has no embedding model.
|
||||
# Note: embedding requires litellm_proxy prefix
|
||||
EMBEDDING_MODEL=litellm_proxy/BAAI/bge-m3
|
||||
LITELLM_PROXY_API_KEY=<replace_with_your_siliconflow_api_key>
|
||||
LITELLM_PROXY_API_BASE=https://api.siliconflow.cn/v1
|
||||
|
||||
Necessary parameters include:
|
||||
|
||||
- `CHAT_MODEL`: The model name of the chat model.
|
||||
|
||||
- `EMBEDDING_MODEL`: The model name of the embedding model.
|
||||
|
||||
- `OPENAI_API_BASE`: The base URL of the API. If `EMBEDDING_MODEL` does not start with `litellm_proxy/`, this is used for both chat and embedding models; otherwise, it is used for `CHAT_MODEL` only.
|
||||
|
||||
Optional parameters (required if your embedding model is provided by a different provider than `CHAT_MODEL`):
|
||||
|
||||
- `LITELLM_PROXY_API_KEY`: The API key for the embedding model, required if `EMBEDDING_MODEL` starts with `litellm_proxy/`.
|
||||
|
||||
- `LITELLM_PROXY_API_BASE`: The base URL for the embedding model, required if `EMBEDDING_MODEL` starts with `litellm_proxy/`.
|
||||
|
||||
**Note:** If you are using an embedding model from a provider different from the chat model, remember to add the `litellm_proxy/` prefix to the `EMBEDDING_MODEL` name.
|
||||
|
||||
|
||||
The `CHAT_MODEL` and `EMBEDDING_MODEL` parameters will be passed into LiteLLM's completion function.
|
||||
|
||||
Therefore, when utilizing models provided by different providers, first review the interface configuration of LiteLLM. The model names must match those allowed by LiteLLM.
|
||||
|
||||
Additionally, you need to set up the the additional parameters for the respective model provider, and the parameter names must align with those required by LiteLLM.
|
||||
|
||||
For example, if you are using a DeepSeek model, you need to set as follows:
|
||||
|
||||
.. code-block:: Properties
|
||||
|
||||
# For some models LiteLLM requires a prefix to the model name.
|
||||
CHAT_MODEL=deepseek/deepseek-chat
|
||||
DEEPSEEK_API_KEY=<replace_with_your_deepseek_api_key>
|
||||
|
||||
Besides, when you are using reasoning models, the response might include the thought process. For this case, you need to set the following environment variable:
|
||||
|
||||
.. code-block:: Properties
|
||||
|
||||
REASONING_THINK_RM=True
|
||||
|
||||
For more details on LiteLLM requirements, refer to the `official LiteLLM documentation <https://docs.litellm.ai/docs>`_.
|
||||
|
||||
Configuration Example 2: Azure OpenAI Setup
|
||||
-------------------------------------------
|
||||
Here’s a sample configuration specifically for Azure OpenAI, based on the `official LiteLLM documentation <https://docs.litellm.ai/docs>`_:
|
||||
|
||||
If you're using Azure OpenAI, below is a working example using the Python SDK, following the `LiteLLM Azure OpenAI documentation <https://docs.litellm.ai/docs/providers/azure/>`_:
|
||||
|
||||
.. code-block:: Properties
|
||||
|
||||
from litellm import completion
|
||||
import os
|
||||
|
||||
# Set Azure OpenAI environment variables
|
||||
os.environ["AZURE_API_KEY"] = "<your_azure_api_key>"
|
||||
os.environ["AZURE_API_BASE"] = "<your_azure_api_base>"
|
||||
os.environ["AZURE_API_VERSION"] = "<version>"
|
||||
|
||||
# Make a request to your Azure deployment
|
||||
response = completion(
|
||||
"azure/<your_deployment_name>",
|
||||
messages = [{ "content": "Hello, how are you?", "role": "user" }]
|
||||
)
|
||||
|
||||
To align with the Python SDK example above, you can configure the `CHAT_MODEL` based on the `response` model setting and use the corresponding `os.environ` variables by writing them into your local `.env` file as follows:
|
||||
|
||||
.. code-block:: Properties
|
||||
|
||||
cat << EOF > .env
|
||||
# CHAT MODEL: Azure OpenAI via LiteLLM
|
||||
CHAT_MODEL=azure/<your_deployment_name>
|
||||
AZURE_API_BASE=https://<your_azure_base>.openai.azure.com/
|
||||
AZURE_API_KEY=<your_azure_api_key>
|
||||
AZURE_API_VERSION=<version>
|
||||
|
||||
# EMBEDDING MODEL: Using SiliconFlow via litellm_proxy
|
||||
EMBEDDING_MODEL=litellm_proxy/BAAI/bge-large-en-v1.5
|
||||
LITELLM_PROXY_API_KEY=<your_siliconflow_api_key>
|
||||
LITELLM_PROXY_API_BASE=https://api.siliconflow.cn/v1
|
||||
EOF
|
||||
|
||||
This configuration allows you to call Azure OpenAI through LiteLLM while using an external provider (e.g., SiliconFlow) for embeddings.
|
||||
|
||||
If your `Azure OpenAI API Key`` supports `embedding model`, you can refer to the following configuration example.
|
||||
|
||||
.. code-block:: Properties
|
||||
|
||||
cat << EOF > .env
|
||||
EMBEDDING_MODEL=azure/<Model deployment supporting embedding>
|
||||
CHAT_MODEL=azure/<your deployment name>
|
||||
AZURE_API_KEY=<replace_with_your_openai_api_key>
|
||||
AZURE_API_BASE=<your_unified_api_base>
|
||||
AZURE_API_VERSION=<azure api version>
|
||||
|
||||
Execution Environment Configuration
|
||||
===================================
|
||||
|
||||
Coder Environment Configuration (Docker vs. Conda)
|
||||
|
||||
RD-Agent's coders can execute code in different environments. You can control this behavior by setting environment variables in your ``.env`` file. This is useful for switching between a local Conda environment and an isolated Docker container.
|
||||
|
||||
To configure the environment, add the corresponding line to your ``.env`` file based on the scenario you are running.
|
||||
|
||||
**For the Model (Quant) Scenario:**
|
||||
|
||||
The execution environment is determined by the ``MODEL_COSTEER_ENV_TYPE`` variable, which is read from ``rdagent/components/coder/model_coder/conf.py``.
|
||||
|
||||
* **To use Docker** (recommended for isolated execution):
|
||||
|
||||
.. code-block:: properties
|
||||
|
||||
MODEL_COSTEER_ENV_TYPE=docker
|
||||
|
||||
* **To use Conda** (for running in a local Conda environment):
|
||||
|
||||
.. code-block:: properties
|
||||
|
||||
MODEL_COSTEER_ENV_TYPE=conda
|
||||
|
||||
**For the Data Science Scenario:**
|
||||
|
||||
The execution environment is determined by the ``DS_CODER_COSTEER_ENV_TYPE`` variable, which is read from ``rdagent/components/coder/data_science/conf.py``.
|
||||
|
||||
* **To use Docker** (recommended for isolated execution):
|
||||
|
||||
.. code-block:: properties
|
||||
|
||||
DS_CODER_COSTEER_ENV_TYPE=docker
|
||||
|
||||
* **To use Conda** (for running in a local Conda environment):
|
||||
|
||||
.. code-block:: properties
|
||||
|
||||
DS_CODER_COSTEER_ENV_TYPE=conda
|
||||
|
||||
|
||||
Custom Time Segment Configuration (Train / Valid / Test)
|
||||
=========================================================
|
||||
|
||||
RD-Agent now supports user-defined time segments for training, validation,
|
||||
and testing (backtesting). Users can customize these segments via environment
|
||||
variables in the ``.env`` file, depending on the scenario being executed.
|
||||
|
||||
This feature allows greater flexibility when running experiments on different
|
||||
time ranges without modifying code or YAML configurations.
|
||||
|
||||
Fin-Factor Scenario
|
||||
-------------------
|
||||
|
||||
When running the **fin_factor** scenario, you can configure the time segments
|
||||
using the following environment variables. These variables are read by the
|
||||
Factor-related PropSettings and directly affect the execution process.
|
||||
|
||||
Add the following entries to your ``.env`` file as needed:
|
||||
|
||||
.. code-block:: properties
|
||||
|
||||
QLIB_FACTOR_TRAIN_START=<train start date, default is 2008-01-01>
|
||||
QLIB_FACTOR_TRAIN_END=<train end date, default is 2014-12-31>
|
||||
QLIB_FACTOR_VALID_START=<valid start date, default is 2015-01-01>
|
||||
QLIB_FACTOR_VALID_END=<valid end date, default is 2016-12-31>
|
||||
QLIB_FACTOR_TEST_START=<test / backtest start date, default is 2017-01-01>
|
||||
QLIB_FACTOR_TEST_END=<test / backtest end date, default is 2020-12-31>
|
||||
|
||||
Fin-Model Scenario
|
||||
------------------
|
||||
|
||||
When running the **fin_model** scenario, the model training, validation, and
|
||||
testing time segments can be configured independently via the following
|
||||
environment variables:
|
||||
|
||||
.. code-block:: properties
|
||||
|
||||
QLIB_MODEL_TRAIN_START=<train start date, default is 2008-01-01>
|
||||
QLIB_MODEL_TRAIN_END=<train end date, default is 2014-12-31>
|
||||
QLIB_MODEL_VALID_START=<valid start date, default is 2015-01-01>
|
||||
QLIB_MODEL_VALID_END=<valid end date, default is 2016-12-31>
|
||||
QLIB_MODEL_TEST_START=<test / backtest start date, default is 2017-01-01>
|
||||
QLIB_MODEL_TEST_END=<test / backtest end date, default is 2020-12-31>
|
||||
|
||||
These settings are used during model training and evaluation and directly
|
||||
impact the execution workflow.
|
||||
|
||||
Fin-Quant Scenario
|
||||
------------------
|
||||
|
||||
When running the **fin_quant** scenario, RD-Agent supports configuring time
|
||||
segments for factor, model, and quant stages simultaneously.
|
||||
|
||||
**Note:** The ``QLIB_QUANT_*`` variables are only used for front-end UI display
|
||||
purposes and do **not** affect the actual execution process.
|
||||
|
||||
You may configure the following variables in your ``.env`` file:
|
||||
|
||||
.. code-block:: properties
|
||||
|
||||
QLIB_FACTOR_TRAIN_START=<train start date, default is 2008-01-01>
|
||||
QLIB_FACTOR_TRAIN_END=<train end date, default is 2014-12-31>
|
||||
QLIB_FACTOR_VALID_START=<valid start date, default is 2015-01-01>
|
||||
QLIB_FACTOR_VALID_END=<valid end date, default is 2016-12-31>
|
||||
QLIB_FACTOR_TEST_START=<test / backtest start date, default is 2017-01-01>
|
||||
QLIB_FACTOR_TEST_END=<test / backtest end date, default is 2020-12-31>
|
||||
|
||||
QLIB_MODEL_TRAIN_START=<train start date, default is 2008-01-01>
|
||||
QLIB_MODEL_TRAIN_END=<train end date, default is 2014-12-31>
|
||||
QLIB_MODEL_VALID_START=<valid start date, default is 2015-01-01>
|
||||
QLIB_MODEL_VALID_END=<valid end date, default is 2016-12-31>
|
||||
QLIB_MODEL_TEST_START=<test / backtest start date, default is 2017-01-01>
|
||||
QLIB_MODEL_TEST_END=<test / backtest end date, default is 2020-12-31>
|
||||
|
||||
QLIB_QUANT_TRAIN_START=<train start date, default is 2008-01-01>
|
||||
QLIB_QUANT_TRAIN_END=<train end date, default is 2014-12-31>
|
||||
QLIB_QUANT_VALID_START=<valid start date, default is 2015-01-01>
|
||||
QLIB_QUANT_VALID_END=<valid end date, default is 2016-12-31>
|
||||
QLIB_QUANT_TEST_START=<test / backtest start date, default is 2017-01-01>
|
||||
QLIB_QUANT_TEST_END=<test / backtest end date, default is 2020-12-31>
|
||||
|
||||
This setup allows the front-end to display consistent segment information
|
||||
across different stages while keeping execution logic unchanged.
|
||||
|
||||
|
||||
Configuration(deprecated)
|
||||
=========================
|
||||
|
||||
To run the application, please create a `.env` file in the root directory of the project and add environment variables according to your requirements.
|
||||
|
||||
If you are using this deprecated version, you should set `BACKEND` to `rdagent.oai.backend.DeprecBackend`.
|
||||
|
||||
.. code-block:: Properties
|
||||
|
||||
BACKEND=rdagent.oai.backend.DeprecBackend
|
||||
|
||||
Here are some other configuration options that you can use:
|
||||
|
||||
OpenAI API
|
||||
------------
|
||||
|
||||
Here is a standard configuration for the user using the OpenAI API.
|
||||
|
||||
.. code-block:: Properties
|
||||
|
||||
OPENAI_API_KEY=<your_api_key>
|
||||
EMBEDDING_MODEL=text-embedding-3-small
|
||||
CHAT_MODEL=gpt-4-turbo
|
||||
|
||||
Azure OpenAI
|
||||
------------
|
||||
|
||||
The following environment variables are standard configuration options for the user using the OpenAI API.
|
||||
|
||||
.. code-block:: Properties
|
||||
|
||||
USE_AZURE=True
|
||||
|
||||
EMBEDDING_OPENAI_API_KEY=<replace_with_your_azure_openai_api_key>
|
||||
EMBEDDING_AZURE_API_BASE= # The endpoint for the Azure OpenAI API.
|
||||
EMBEDDING_AZURE_API_VERSION= # The version of the Azure OpenAI API.
|
||||
EMBEDDING_MODEL=text-embedding-3-small
|
||||
|
||||
CHAT_OPENAI_API_KEY=<replace_with_your_azure_openai_api_key>
|
||||
CHAT_AZURE_API_BASE= # The endpoint for the Azure OpenAI API.
|
||||
CHAT_AZURE_API_VERSION= # The version of the Azure OpenAI API.
|
||||
CHAT_MODEL= # The model name of the Azure OpenAI API.
|
||||
|
||||
Use Azure Token Provider
|
||||
------------------------
|
||||
|
||||
If you are using the Azure token provider, you need to set the `CHAT_USE_AZURE_TOKEN_PROVIDER` and `EMBEDDING_USE_AZURE_TOKEN_PROVIDER` environment variable to `True`. then
|
||||
use the environment variables provided in the `Azure Configuration section <installation_and_configuration.html#azure-openai>`_.
|
||||
|
||||
|
||||
☁️ Azure Configuration
|
||||
- Install Azure CLI:
|
||||
|
||||
```sh
|
||||
curl -L https://aka.ms/InstallAzureCli | bash
|
||||
```
|
||||
|
||||
- Log in to Azure:
|
||||
|
||||
```sh
|
||||
az login --use-device-code
|
||||
```
|
||||
|
||||
- `exit` and re-login to your environment (this step may not be necessary).
|
||||
|
||||
|
||||
Configuration List
|
||||
------------------
|
||||
|
||||
.. TODO: use `autodoc-pydantic` .
|
||||
|
||||
- OpenAI API Setting
|
||||
|
||||
+-----------------------------------+-----------------------------------------------------------------+-------------------------+
|
||||
| Configuration Option | Meaning | Default Value |
|
||||
+===================================+=================================================================+=========================+
|
||||
| OPENAI_API_KEY | API key for both chat and embedding models | None |
|
||||
+-----------------------------------+-----------------------------------------------------------------+-------------------------+
|
||||
| EMBEDDING_OPENAI_API_KEY | Use a different API key for embedding model | None |
|
||||
+-----------------------------------+-----------------------------------------------------------------+-------------------------+
|
||||
| CHAT_OPENAI_API_KEY | Set to use a different API key for chat model | None |
|
||||
+-----------------------------------+-----------------------------------------------------------------+-------------------------+
|
||||
| EMBEDDING_MODEL | Name of the embedding model | text-embedding-3-small |
|
||||
+-----------------------------------+-----------------------------------------------------------------+-------------------------+
|
||||
| CHAT_MODEL | Name of the chat model | gpt-4-turbo |
|
||||
+-----------------------------------+-----------------------------------------------------------------+-------------------------+
|
||||
| EMBEDDING_AZURE_API_BASE | Base URL for the Azure OpenAI API | None |
|
||||
+-----------------------------------+-----------------------------------------------------------------+-------------------------+
|
||||
| EMBEDDING_AZURE_API_VERSION | Version of the Azure OpenAI API | None |
|
||||
+-----------------------------------+-----------------------------------------------------------------+-------------------------+
|
||||
| CHAT_AZURE_API_BASE | Base URL for the Azure OpenAI API | None |
|
||||
+-----------------------------------+-----------------------------------------------------------------+-------------------------+
|
||||
| CHAT_AZURE_API_VERSION | Version of the Azure OpenAI API | None |
|
||||
+-----------------------------------+-----------------------------------------------------------------+-------------------------+
|
||||
| USE_AZURE | True if you are using Azure OpenAI | False |
|
||||
+-----------------------------------+-----------------------------------------------------------------+-------------------------+
|
||||
| CHAT_USE_AZURE_TOKEN_PROVIDER | True if you are using an Azure Token Provider in chat model | False |
|
||||
+-----------------------------------+-----------------------------------------------------------------+-------------------------+
|
||||
| EMBEDDING_USE_AZURE_TOKEN_PROVIDER| True if you are using an Azure Token Provider in embedding model| False |
|
||||
+-----------------------------------+-----------------------------------------------------------------+-------------------------+
|
||||
|
||||
- Globol Setting
|
||||
|
||||
+-----------------------------+--------------------------------------------------+-------------------------+
|
||||
| Configuration Option | Meaning | Default Value |
|
||||
+=============================+==================================================+=========================+
|
||||
| max_retry | Maximum number of times to retry | 10 |
|
||||
+-----------------------------+--------------------------------------------------+-------------------------+
|
||||
| retry_wait_seconds | Number of seconds to wait before retrying | 1 |
|
||||
+-----------------------------+--------------------------------------------------+-------------------------+
|
||||
+ log_trace_path | Path to log trace file | None |
|
||||
+-----------------------------+--------------------------------------------------+-------------------------+
|
||||
+ log_llm_chat_content | Flag to indicate if chat content is logged | True |
|
||||
+-----------------------------+--------------------------------------------------+-------------------------+
|
||||
|
||||
|
||||
- Cache Setting
|
||||
|
||||
.. TODO: update Meaning for caches
|
||||
|
||||
+------------------------------+--------------------------------------------------+-------------------------+
|
||||
| Configuration Option | Meaning | Default Value |
|
||||
+==============================+==================================================+=========================+
|
||||
| dump_chat_cache | Flag to indicate if chat cache is dumped | False |
|
||||
+------------------------------+--------------------------------------------------+-------------------------+
|
||||
| dump_embedding_cache | Flag to indicate if embedding cache is dumped | False |
|
||||
+------------------------------+--------------------------------------------------+-------------------------+
|
||||
| use_chat_cache | Flag to indicate if chat cache is used | False |
|
||||
+------------------------------+--------------------------------------------------+-------------------------+
|
||||
| use_embedding_cache | Flag to indicate if embedding cache is used | False |
|
||||
+------------------------------+--------------------------------------------------+-------------------------+
|
||||
| prompt_cache_path | Path to prompt cache | ./prompt_cache.db |
|
||||
+------------------------------+--------------------------------------------------+-------------------------+
|
||||
| max_past_message_include | Maximum number of past messages to include | 10 |
|
||||
+------------------------------+--------------------------------------------------+-------------------------+
|
||||
|
||||
|
||||
|
||||
|
||||
Loading Configuration
|
||||
---------------------
|
||||
|
||||
For users' convenience, we provide a CLI interface called `rdagent`, which automatically runs `load_dotenv()` to load environment variables from the `.env` file.
|
||||
However, this feature is not enabled by default for other scripts. We recommend users load the environment with the following steps:
|
||||
|
||||
|
||||
- ⚙️ Environment Configuration
|
||||
- Place the `.env` file in the same directory as the `.env.example` file.
|
||||
- The `.env.example` file contains the environment variables required for users using the OpenAI API (Please note that `.env.example` is an example file. `.env` is the one that will be finally used.)
|
||||
|
||||
- Export each variable in the .env file:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
export $(grep -v '^#' .env | xargs)
|
||||
|
||||
- If you want to change the default environment variables, you can refer to the above configuration and edith the `.env` file.
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
=========================
|
||||
Introduction
|
||||
=========================
|
||||
|
||||
|
||||
|
||||
In modern industry, research and development (R&D) is crucial for the enhancement of industrial productivity, especially in the AI era, where the core aspects of R&D are mainly focused on data and models. We are committed to automate these high-value generic R&D processes through our open source R&D automation tool RDAgent, which let AI drive data-driven AI.
|
||||
|
||||
.. image:: _static/scen.png
|
||||
:alt: Our focused scenario
|
||||
|
||||
|
||||
Our RDAgent is designed to automate the most critical industrial R&D processes, focusing first on data-driven scenarios, to greatly boost the development productivity of models and data.
|
||||
|
||||
Methodologically, we propose an autonomous agent framework that consists of two key parts: (R)esearch stands for actively exploring by proposing new ideas, and (D)evelopment stands for realizing these ideas. The effectiveness of these two components will ultimately get feedbacks through practice, and both research and development capabilities can continuously learn and grow in the process.
|
||||
|
||||
|
||||
For a quick start, visit `our GitHub home page <https://github.com/microsoft/RD-Agent>`_ ⚡. If you've already checked it out and want more details, please keep reading.
|
||||
@@ -0,0 +1,35 @@
|
||||
@ECHO OFF
|
||||
|
||||
pushd %~dp0
|
||||
|
||||
REM Command file for Sphinx documentation
|
||||
|
||||
if "%SPHINXBUILD%" == "" (
|
||||
set SPHINXBUILD=sphinx-build
|
||||
)
|
||||
set SOURCEDIR=.
|
||||
set BUILDDIR=build
|
||||
|
||||
%SPHINXBUILD% >NUL 2>NUL
|
||||
if errorlevel 9009 (
|
||||
echo.
|
||||
echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
|
||||
echo.installed, then set the SPHINXBUILD environment variable to point
|
||||
echo.to the full path of the 'sphinx-build' executable. Alternatively you
|
||||
echo.may add the Sphinx directory to PATH.
|
||||
echo.
|
||||
echo.If you don't have Sphinx installed, grab it from
|
||||
echo.https://www.sphinx-doc.org/
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
if "%1" == "" goto help
|
||||
|
||||
%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
|
||||
goto end
|
||||
|
||||
:help
|
||||
%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
|
||||
|
||||
:end
|
||||
popd
|
||||
@@ -0,0 +1,24 @@
|
||||
======
|
||||
Policy
|
||||
======
|
||||
|
||||
This project welcomes contributions and suggestions. Most contributions require you to agree to a
|
||||
Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us
|
||||
the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com.
|
||||
|
||||
When you submit a pull request, a CLA bot will automatically determine whether you need to provide
|
||||
a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions
|
||||
provided by the bot. You will only need to do this once across all repos using our CLA.
|
||||
|
||||
This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
|
||||
For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or
|
||||
contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.
|
||||
|
||||
Trademarks
|
||||
==========
|
||||
|
||||
This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft
|
||||
trademarks or logos is subject to and must follow
|
||||
[Microsoft's Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks/usage/general).
|
||||
Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship.
|
||||
Any use of third-party trademarks or logos are subject to those third-party's policies.
|
||||
@@ -0,0 +1,27 @@
|
||||
===============================
|
||||
Framework Design & Components
|
||||
===============================
|
||||
|
||||
Framework & Components
|
||||
=========================
|
||||
|
||||
.. NOTE: This depends on the correctness of `c-v` of github.
|
||||
|
||||
.. image:: _static/Framework-RDAgent.png
|
||||
:alt: Components & Feature Level
|
||||
|
||||
The image above shows the overall framework of RDAgent.
|
||||
|
||||
In a data mining expert's daily research and development process, they propose a hypothesis (e.g., a model structure like RNN can capture patterns in time-series data), design experiments (e.g., finance data contains time-series and we can verify the hypothesis in this scenario), implement the experiment as code (e.g., Pytorch model structure), and then execute the code to get feedback (e.g., metrics, loss curve, etc.). The experts learn from the feedback and improve in the next iteration.
|
||||
|
||||
We have established a basic method framework that continuously proposes hypotheses, verifies them, and gets feedback from the real world. This is the first scientific research automation framework that supports linking with real-world verification.
|
||||
|
||||
|
||||
.. image:: https://github.com/user-attachments/assets/60cc2712-c32a-4492-a137-8aec59cdc66e
|
||||
:alt: Class Level Figure
|
||||
|
||||
The figure above shows the main classes and how they fit into the workflow for those interested in the detailed code.
|
||||
|
||||
|
||||
.. Detailed Design
|
||||
.. ===============
|
||||
@@ -0,0 +1,4 @@
|
||||
sphinx
|
||||
sphinx_rtd_theme
|
||||
furo
|
||||
importlib.metadata
|
||||
@@ -0,0 +1,109 @@
|
||||
==============================
|
||||
Benchmark
|
||||
==============================
|
||||
|
||||
Introduction
|
||||
=============
|
||||
|
||||
Benchmarking the capabilities of R&D is a crucial research problem in this area. We are continuously exploring methods to benchmark these capabilities. The current benchmarks are listed on this page.
|
||||
|
||||
Development Capability Benchmarking
|
||||
===================================
|
||||
|
||||
Benchmarking is used to evaluate the effectiveness of factors with fixed data. It mainly includes the following steps:
|
||||
|
||||
1. :ref:`read and prepare the eval_data <data>`
|
||||
|
||||
2. :ref:`declare the method to be tested and pass the arguments <config>`
|
||||
|
||||
3. :ref:`declare the eval method and pass the arguments <config>`
|
||||
|
||||
4. :ref:`run the eval <run>`
|
||||
|
||||
5. :ref:`save and show the result <show>`
|
||||
|
||||
Configuration
|
||||
-------------
|
||||
.. _config:
|
||||
|
||||
.. autopydantic_settings:: rdagent.components.benchmark.conf.BenchmarkSettings
|
||||
|
||||
Example
|
||||
+++++++
|
||||
.. _example:
|
||||
|
||||
The default value for ``bench_test_round`` is 10, which takes about 2 hours to run. To modify it from ``10`` to ``2``, adjust the environment variables in the .env file as shown below.
|
||||
|
||||
.. code-block:: Properties
|
||||
|
||||
BENCHMARK_BENCH_TEST_ROUND=2
|
||||
|
||||
Data Format
|
||||
-------------
|
||||
.. _data:
|
||||
|
||||
The sample data in ``bench_data_path`` is a dictionary where each key represents a factor name. The value associated with each key is factor data containing the following information:
|
||||
|
||||
- **description**: A textual description of the factor.
|
||||
- **formulation**: A LaTeX formula representing the model's formulation.
|
||||
- **variables**: A dictionary of variables involved in the factor.
|
||||
- **Category**: The category or classification of the factor.
|
||||
- **Difficulty**: The difficulty level of implementing or understanding the factor.
|
||||
- **gt_code**: A piece of code associated with the factor.
|
||||
|
||||
Here is an example of this data format:
|
||||
|
||||
.. literalinclude:: ../../rdagent/components/benchmark/example.json
|
||||
:language: json
|
||||
|
||||
Ensure the data is placed in the ``FACTOR_COSTEER_SETTINGS.data_folder_debug``. The data files should be in ``.h5`` or ``.md`` format and must not be stored in any subfolders. LLM-Agents will review the file content and implement the tasks.
|
||||
|
||||
.. TODO: Add a script to automatically generate the data in the `rdagent/app/quant_factor_benchmark/data` folder.
|
||||
|
||||
Run Benchmark
|
||||
-------------
|
||||
.. _run:
|
||||
|
||||
Start the benchmark after completing the :doc:`../installation_and_configuration`.
|
||||
|
||||
.. code-block:: Properties
|
||||
|
||||
dotenv run -- python rdagent/app/benchmark/factor/eval.py
|
||||
|
||||
Once completed, a pkl file will be generated, and its path will be printed on the last line of the console.
|
||||
|
||||
Show Result
|
||||
-------------
|
||||
.. _show:
|
||||
|
||||
The ``analysis.py`` script reads data from the pkl file and converts it to an image. Modify the Python code in ``rdagent/app/quant_factor_benchmark/analysis.py`` to specify the path to the pkl file and the output path for the png file.
|
||||
|
||||
.. code-block:: Properties
|
||||
|
||||
dotenv run -- python rdagent/app/benchmark/factor/analysis.py <log/path to.pkl>
|
||||
|
||||
A png file will be saved to the designated path as shown below.
|
||||
|
||||
.. image:: ../_static/benchmark.png
|
||||
|
||||
Related Paper
|
||||
-------------
|
||||
|
||||
- `Towards Data-Centric Automatic R&D <https://arxiv.org/abs/2404.11276>`_:
|
||||
We have developed a comprehensive benchmark called RD2Bench to assess data and model R&D capabilities. This benchmark includes a series of tasks that outline the features or structures of models. These tasks are used to evaluate the ability of LLM-Agents to implement them.
|
||||
|
||||
.. code-block:: bibtex
|
||||
|
||||
@misc{chen2024datacentric,
|
||||
title={Towards Data-Centric Automatic R&D},
|
||||
author={Haotian Chen and Xinjie Shen and Zeqi Ye and Wenjun Feng and Haoxue Wang and Xiao Yang and Xu Yang and Weiqing Liu and Jiang Bian},
|
||||
year={2024},
|
||||
eprint={2404.11276},
|
||||
archivePrefix={arXiv},
|
||||
primaryClass={cs.AI}
|
||||
}
|
||||
|
||||
.. image:: https://github.com/user-attachments/assets/494f55d3-de9e-4e73-ba3d-a787e8f9e841
|
||||
|
||||
To replicate the benchmark detailed in the paper, please consult the factors listed in the following file: `RD2bench.json <../_static/RD2bench.json>`_.
|
||||
Please note use ``only_correct_format=False`` when evaluating the results.
|
||||
@@ -0,0 +1,34 @@
|
||||
===========
|
||||
Research
|
||||
===========
|
||||
|
||||
To achieve the good effects and improve R&D capabilities, we face multiple challenges, the most important of which is the continuous evolution capability. Existing large language models (LLMs) find it difficult to continue growing their capabilities after training is completed. Moreover, the training process of LLMs focuses more on general knowledge, and the lack of depth in more specialized knowledge becomes an obstacle to solving professional R&D problems within the industry. This specialized knowledge needs to be learned and acquired from in-depth industry practice.
|
||||
|
||||
|
||||
Our RD-Agent, on the other hand, can continuously acquire in-depth domain knowledge through deep exploration during the R&D phase, allowing its R&D capabilities to keep growing.
|
||||
|
||||
To address these key challenges and achieve industrial value, a series of research work needs to be completed.
|
||||
|
||||
|
||||
.. list-table:: Research Areas and Descriptions
|
||||
:header-rows: 1
|
||||
|
||||
* - Research Area
|
||||
- Description
|
||||
* - :doc:`Benchmark <benchmark>`
|
||||
- Benchmark the R&D abilities
|
||||
* - Research
|
||||
- Idea proposal: Explore new ideas or refine existing ones
|
||||
* - :doc:`Development <dev>`
|
||||
- Ability to realize ideas: Implement and execute ideas
|
||||
|
||||
|
||||
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
:caption: Doctree:
|
||||
:hidden:
|
||||
|
||||
benchmark
|
||||
dev
|
||||
@@ -0,0 +1,25 @@
|
||||
==============================
|
||||
Development
|
||||
==============================
|
||||
|
||||
|
||||
Related Paper
|
||||
-------------
|
||||
|
||||
- `Collaborative Evolving Strategy for Automatic Data-Centric Development <https://arxiv.org/abs/2407.18690>`_
|
||||
Co-STEER is a method to tackle data-centric development (AD2) tasks and highlight its main challenges, which need expert-like implementation (i.e., learning domain knowledge from practice) and task scheduling capability (e.g., starting with easier tasks for better overall efficiency), areas that previous work has largely overlooked. Our Co-STEER agent enhances its domain knowledge through our evolving strategy and improves both its scheduling and implementation skills by gathering and using domain-specific practical experience. With a better schedule, implementation becomes faster. At the same time, as implementation feedback becomes more detailed, scheduling accuracy improves. These two capabilities grow together through practical feedback, enabling a collaborative evolution process.
|
||||
|
||||
.. code-block:: bibtex
|
||||
|
||||
@misc{yang2024collaborative,
|
||||
title={Collaborative Evolving Strategy for Automatic Data-Centric Development},
|
||||
author={Xu Yang and Haotian Chen and Wenjun Feng and Haoxue Wang and Zeqi Ye and Xinjie Shen and Xiao Yang and Shizhao Sun and Weiqing Liu and Jiang Bian},
|
||||
year={2024},
|
||||
eprint={2407.18690},
|
||||
archivePrefix={arXiv},
|
||||
primaryClass={cs.AI}
|
||||
}
|
||||
|
||||
.. image:: https://github.com/user-attachments/assets/75d9769b-0edd-4caf-9d45-57d1e577054b
|
||||
:alt: Collaborative Evolving Strategy for Automatic Data-Centric Development
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
=========================
|
||||
Scenarios
|
||||
=========================
|
||||
|
||||
Scenario lists
|
||||
=========================
|
||||
|
||||
In the two key areas of data-driven scenarios, model implementation and data building, our system aims to serve two main roles: 🦾copilot and 🤖agent.
|
||||
|
||||
- The 🦾copilot follows human instructions to automate repetitive tasks.
|
||||
- The 🤖agent, being more autonomous, actively proposes ideas for better results in the future.
|
||||
|
||||
The supported scenarios are listed below:
|
||||
|
||||
|
||||
.. list-table::
|
||||
:header-rows: 1
|
||||
|
||||
* - Scenario/Target
|
||||
- Model Implementation
|
||||
- Data Building
|
||||
* - 💹 Finance
|
||||
- :ref:`🥇The First Data-Centric Quant Multi-Agent Framework <quant_agent_fin>`
|
||||
- :ref:`🤖Iteratively Proposing Ideas & Evolving <model_agent_fin>`
|
||||
|
||||
:ref:`🦾Auto reports reading & implementation <data_copilot_fin>`
|
||||
|
||||
:ref:`🤖Iteratively Proposing Ideas & Evolving <data_agent_fin>`
|
||||
* - 🏭 General
|
||||
- :ref:`🦾Auto paper reading & implementation <model_copilot_general>`
|
||||
|
||||
:ref:`🧪FT-Agent for LLM fine-tuning <finetune_agent>`
|
||||
- :ref:`🤖 Data Science <data_science_agent>`
|
||||
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
:caption: Doctree:
|
||||
:hidden:
|
||||
|
||||
quant_agent_fin
|
||||
data_agent_fin
|
||||
data_copilot_fin
|
||||
model_agent_fin
|
||||
model_copilot_general
|
||||
data_science
|
||||
finetune
|
||||
@@ -0,0 +1,138 @@
|
||||
.. _data_agent_fin:
|
||||
|
||||
=====================
|
||||
Finance Data Agent
|
||||
=====================
|
||||
|
||||
|
||||
**🤖 Automated Quantitative Trading & Iterative Factors Evolution**
|
||||
-------------------------------------------------------------------
|
||||
|
||||
📖 Background
|
||||
~~~~~~~~~~~~~~
|
||||
In the dynamic world of quantitative trading, **factors** serve as the strategic tools that enable traders to exploit market inefficiencies.
|
||||
These factors—ranging from simple metrics like price-to-earnings ratios to complex models like discounted cash flows—are the key to predicting stock prices with a high degree of accuracy.
|
||||
|
||||
By leveraging these factors, quantitative traders can develop sophisticated strategies that not only identify market patterns but also significantly enhance trading efficiency and precision.
|
||||
The ability to systematically analyze and apply these factors is what separates ordinary trading from truly strategic market outmaneuvering.
|
||||
And this is where the **Finance Model Agent** comes into play.
|
||||
|
||||
🎥 `Demo <https://rdagent.azurewebsites.net/factor_loop>`_
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. raw:: html
|
||||
|
||||
<div style="display: flex; justify-content: center; align-items: center;">
|
||||
<video width="600" controls>
|
||||
<source src="https://rdagent.azurewebsites.net/media/65bb598f1372c1857ccbf09b2acf5d55830911625048c03102291098.mp4" type="video/mp4">
|
||||
Your browser does not support the video tag.
|
||||
</video>
|
||||
</div>
|
||||
|
||||
|
||||
🌟 Introduction
|
||||
~~~~~~~~~~~~~~~~
|
||||
In this scenario, our agent illustrates the iterative process of hypothesis generation, knowledge construction, and decision-making.
|
||||
|
||||
It highlights how financial factors evolve through continuous feedback and refinement.
|
||||
|
||||
Here's an enhanced outline of the steps:
|
||||
|
||||
**Step 1 : Hypothesis Generation 🔍**
|
||||
|
||||
- Generate and propose initial hypotheses based on previous experiment analysis and domain expertise, with thorough reasoning and financial justification.
|
||||
|
||||
**Step 2 : Factor Creation ✨**
|
||||
|
||||
- Based on the hypothesis, divide the tasks.
|
||||
- Each task involves developing, defining, and implementing a new financial factor, including its name, description, formulation, and variables.
|
||||
|
||||
**Step 3 : Factor Implementation 👨💻**
|
||||
|
||||
- Implement the factor code based on the description, evolving it as a developer would.
|
||||
- Quantitatively validate the newly created factors.
|
||||
|
||||
**Step 4 : Backtesting with Qlib 📉**
|
||||
|
||||
- Integrate the full dataset into the factor implementation code and prepare the factor library.
|
||||
- Conduct backtesting using the Alpha158 plus newly developed factors and LGBModel in Qlib to evaluate the new factors' effectiveness and performance.
|
||||
|
||||
+----------------+------------+----------------+----------------------------------------------------+
|
||||
| Dataset | Model | Factors | Data Split |
|
||||
+================+============+================+====================================================+
|
||||
| CSI300 | LGBModel | Alpha158 Plus | +-----------+--------------------------+ |
|
||||
| | | | | Train | 2008-01-01 to 2014-12-31 | |
|
||||
| | | | +-----------+--------------------------+ |
|
||||
| | | | | Valid | 2015-01-01 to 2016-12-31 | |
|
||||
| | | | +-----------+--------------------------+ |
|
||||
| | | | | Test | 2017-01-01 to 2020-08-01 | |
|
||||
| | | | +-----------+--------------------------+ |
|
||||
+----------------+------------+----------------+----------------------------------------------------+
|
||||
|
||||
|
||||
**Step 5 : Feedback Analysis 🔍**
|
||||
|
||||
- Analyze backtest results to assess performance.
|
||||
- Incorporate feedback to refine hypotheses and improve the model.
|
||||
|
||||
**Step 6 :Hypothesis Refinement ♻️**
|
||||
|
||||
- Refine hypotheses based on feedback from backtesting.
|
||||
- Repeat the process to continuously improve the model.
|
||||
|
||||
⚡ Quick Start
|
||||
~~~~~~~~~~~~~~~~~
|
||||
|
||||
Please refer to the installation part in :doc:`../installation_and_configuration` to prepare your system dependency.
|
||||
|
||||
You can try our demo by running the following command:
|
||||
|
||||
- 🐍 Create a Conda Environment
|
||||
|
||||
- Create a new conda environment with Python (3.10 and 3.11 are well tested in our CI):
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
conda create -n rdagent python=3.10
|
||||
|
||||
- Activate the environment:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
conda activate rdagent
|
||||
|
||||
- 📦 Install the RDAgent
|
||||
|
||||
- You can install the RDAgent package from PyPI:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
pip install rdagent
|
||||
|
||||
- 🚀 Run the Application
|
||||
|
||||
- You can directly run the application by using the following command:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
rdagent fin_factor
|
||||
|
||||
|
||||
🛠️ Usage of modules
|
||||
~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. _Env Config:
|
||||
|
||||
- **Env Config**
|
||||
|
||||
The following environment variables can be set in the `.env` file to customize the application's behavior:
|
||||
|
||||
.. autopydantic_settings:: rdagent.app.qlib_rd_loop.conf.FactorBasePropSetting
|
||||
:settings-show-field-summary: False
|
||||
:exclude-members: Config
|
||||
|
||||
.. autopydantic_settings:: rdagent.components.coder.factor_coder.config.FactorCoSTEERSettings
|
||||
:settings-show-field-summary: False
|
||||
:members: coder_use_cache, data_folder, data_folder_debug, file_based_execution_timeout, select_method, max_loop, knowledge_base_path, new_knowledge_base_path
|
||||
:exclude-members: Config, fail_task_trial_limit, v1_query_former_trace_limit, v1_query_similar_success_limit, v2_query_component_limit, v2_query_error_limit, v2_query_former_trace_limit, v2_error_summary, v2_knowledge_sampler
|
||||
:no-index:
|
||||
@@ -0,0 +1,164 @@
|
||||
.. _data_copilot_fin:
|
||||
|
||||
=====================
|
||||
Finance Data Copilot
|
||||
=====================
|
||||
|
||||
|
||||
**🤖 Automated Quantitative Trading & Factors Extraction from Financial Reports**
|
||||
---------------------------------------------------------------------------------
|
||||
|
||||
📖 Background
|
||||
~~~~~~~~~~~~~~
|
||||
**Research reports** are treasure troves of insights, often unveiling potential **factors** that can drive successful quantitative trading strategies.
|
||||
Yet, with the sheer volume of reports available, extracting the most valuable insights efficiently becomes a daunting task.
|
||||
|
||||
Furthermore, rather than hastily replicating factors from a report, it's essential to delve into the underlying logic of their construction.
|
||||
Does the factor capture the essential market dynamics? How unique is it compared to the factors already in your library?
|
||||
|
||||
Therefore, there is an urgent need for a systematic approach to design a framework that can effectively manage this process.
|
||||
And this is where the **Finance Data Copilot** steps in.
|
||||
|
||||
|
||||
🎥 `Demo <https://rdagent.azurewebsites.net/report_factor>`_
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. raw:: html
|
||||
|
||||
<div style="display: flex; justify-content: center; align-items: center;">
|
||||
<video width="600" controls>
|
||||
<source src="https://rdagent.azurewebsites.net/media/7b14b2bd3d8771da9cf7eb799b6d96729cec3d35c8d4f68060f3e2fd.mp4" type="video/mp4">
|
||||
Your browser does not support the video tag.
|
||||
</video>
|
||||
</div>
|
||||
|
||||
|
||||
🌟 Introduction
|
||||
~~~~~~~~~~~~~~~~
|
||||
In this scenario, RDAgent demonstrates the process of extracting factors from financial research reports, implementing these factors, and analyzing their performance through Qlib backtesting.
|
||||
This process continually expands and refines the factor library.
|
||||
|
||||
Here's an enhanced outline of the steps:
|
||||
|
||||
**Step 1 : Hypothesis Generation 🔍**
|
||||
|
||||
- Generate and propose initial hypotheses based on insights from financial reports with thorough reasoning and financial justification.
|
||||
|
||||
**Step 2 : Factor Creation ✨**
|
||||
|
||||
- Based on the hypothesis and financial reports, divide the tasks.
|
||||
- Each task involves developing, defining, and implementing a new financial factor, including its name, description, formulation, and variables.
|
||||
|
||||
**Step 3 : Factor Implementation 👨💻**
|
||||
|
||||
- Implement the factor code based on the description, evolving it as a developer would.
|
||||
- Quantitatively validate the newly created factors.
|
||||
|
||||
**Step 4 : Backtesting with Qlib 📉**
|
||||
|
||||
- Integrate the full dataset into the factor implementation code and prepare the factor library.
|
||||
- Conduct backtesting using the Alpha158 plus newly developed factors and LGBModel in Qlib to evaluate the new factors' effectiveness and performance.
|
||||
|
||||
+----------------+------------+----------------+----------------------------------------------------+
|
||||
| Dataset | Model | Factors | Data Split |
|
||||
+================+============+================+====================================================+
|
||||
| CSI300 | LGBModel | Alpha158 Plus | +-----------+--------------------------+ |
|
||||
| | | | | Train | 2008-01-01 to 2014-12-31 | |
|
||||
| | | | +-----------+--------------------------+ |
|
||||
| | | | | Valid | 2015-01-01 to 2016-12-31 | |
|
||||
| | | | +-----------+--------------------------+ |
|
||||
| | | | | Test | 2017-01-01 to 2020-08-01 | |
|
||||
| | | | +-----------+--------------------------+ |
|
||||
+----------------+------------+----------------+----------------------------------------------------+
|
||||
|
||||
**Step 5 : Feedback Analysis 🔍**
|
||||
|
||||
- Analyze backtest results to assess performance.
|
||||
- Incorporate feedback to refine hypotheses and improve the model.
|
||||
|
||||
**Step 6 :Hypothesis Refinement ♻️**
|
||||
|
||||
- Refine hypotheses based on feedback from backtesting.
|
||||
- Repeat the process to continuously improve the model.
|
||||
|
||||
⚡ Quick Start
|
||||
~~~~~~~~~~~~~~~~~
|
||||
|
||||
Please refer to the installation part in :doc:`../installation_and_configuration` to prepare your system dependency.
|
||||
|
||||
You can try our demo by running the following command:
|
||||
|
||||
- 🐍 Create a Conda Environment
|
||||
|
||||
- Create a new conda environment with Python (3.10 and 3.11 are well tested in our CI):
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
conda create -n rdagent python=3.10
|
||||
|
||||
- Activate the environment:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
conda activate rdagent
|
||||
|
||||
- 📦 Install the RDAgent
|
||||
|
||||
- You can install the RDAgent package from PyPI:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
pip install rdagent
|
||||
|
||||
- 🚀 Run the Application
|
||||
|
||||
- Download the financial reports you wish to extract factors from and store them in your preferred folder.
|
||||
|
||||
- Specifically, you can follow this example, or use your own method:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
wget https://github.com/SunsetWolf/rdagent_resource/releases/download/reports/all_reports.zip
|
||||
unzip all_reports.zip -d git_ignore_folder/reports
|
||||
|
||||
- Run the application with the following command:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
rdagent fin_factor_report --report-folder=git_ignore_folder/reports
|
||||
|
||||
- Alternatively, you can store the paths of the reports in `report_result_json_file_path`. The format should be:
|
||||
|
||||
.. code-block:: json
|
||||
|
||||
[
|
||||
"git_ignore_folder/report/fin_report1.pdf",
|
||||
"git_ignore_folder/report/fin_report2.pdf",
|
||||
"git_ignore_folder/report/fin_report3.pdf"
|
||||
]
|
||||
|
||||
- Then, run the application using the following command:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
rdagent fin_factor_report
|
||||
|
||||
🛠️ Usage of modules
|
||||
~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. _Env Config:
|
||||
|
||||
- **Env Config**
|
||||
|
||||
The following environment variables can be set in the `.env` file to customize the application's behavior:
|
||||
|
||||
.. autopydantic_settings:: rdagent.app.qlib_rd_loop.conf.FactorFromReportPropSetting
|
||||
:settings-show-field-summary: False
|
||||
:show-inheritance:
|
||||
:exclude-members: Config
|
||||
|
||||
.. autopydantic_settings:: rdagent.components.coder.factor_coder.config.FactorCoSTEERSettings
|
||||
:settings-show-field-summary: False
|
||||
:members: coder_use_cache, data_folder, data_folder_debug, file_based_execution_timeout, select_method, max_loop, knowledge_base_path, new_knowledge_base_path
|
||||
:exclude-members: Config, python_bin, fail_task_trial_limit, v1_query_former_trace_limit, v1_query_similar_success_limit, v2_query_component_limit, v2_query_error_limit, v2_query_former_trace_limit, v2_error_summary, v2_knowledge_sampler
|
||||
:no-index:
|
||||
@@ -0,0 +1,566 @@
|
||||
.. _data_science_agent:
|
||||
|
||||
=======================
|
||||
Data Science Agent
|
||||
=======================
|
||||
|
||||
**🤖 Automated Feature Engineering & Model Tuning Evolution**
|
||||
------------------------------------------------------------------------------------------
|
||||
The Data Science Agent is an agent that can automatically perform feature engineering and model tuning. It can be used to solve various data science problems, such as image classification, time series forecasting, and text classification.
|
||||
|
||||
🌟 Introduction
|
||||
~~~~~~~~~~~~~~~~~~
|
||||
|
||||
In this scenario, our automated system proposes hypothesis, choose action, implements code, conducts validation, and utilizes feedback in a continuous, iterative process.
|
||||
|
||||
The goal is to automatically optimize performance metrics within the validation set or Kaggle Leaderboard, ultimately discovering the most efficient features and models through autonomous research and development.
|
||||
|
||||
Here's an enhanced outline of the steps:
|
||||
|
||||
**Step 1 : Hypothesis Generation 🔍**
|
||||
|
||||
- Generate and propose initial hypotheses based on previous experiment analysis and domain expertise, with thorough reasoning and financial justification.
|
||||
|
||||
**Step 2 : Experiment Creation ✨**
|
||||
|
||||
- Transform the hypothesis into a task.
|
||||
- Choose a specific action within feature engineering or model tuning.
|
||||
- Develop, define, and implement a new feature or model, including its name, description, and formulation.
|
||||
|
||||
**Step 3 : Model/Feature Implementation 👨💻**
|
||||
|
||||
- Implement the model code based on the detailed description.
|
||||
- Evolve the model iteratively as a developer would, ensuring accuracy and efficiency.
|
||||
|
||||
**Step 4 : Validation on Test Set or Kaggle 📉**
|
||||
|
||||
- Validate the newly developed model using the test set or Kaggle dataset.
|
||||
- Assess the model's effectiveness and performance based on the validation results.
|
||||
|
||||
**Step 5: Feedback Analysis 🔍**
|
||||
|
||||
- Analyze validation results to assess performance.
|
||||
- Use insights to refine hypotheses and enhance the model.
|
||||
|
||||
**Step 6: Hypothesis Refinement ♻️**
|
||||
|
||||
- Adjust hypotheses based on validation feedback.
|
||||
- Iterate the process to continuously improve the model.
|
||||
|
||||
📖 Data Science Background
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
In the evolving landscape of artificial intelligence, **Data Science** represents a powerful paradigm where machines engage in autonomous exploration, hypothesis testing, and model development across diverse domains — from healthcare and finance to logistics and research.
|
||||
|
||||
The **Data Science** Agent stands as a central engine in this transformation, enabling users to automate the entire machine learning workflow: from hypothesis generation to code implementation, validation, and refinement — all guided by performance feedback.
|
||||
|
||||
By leveraging the **Data Science** Agent, researchers and developers can accelerate experimentation cycles. Whether fine-tuning custom models or competing in high-stakes benchmarks like Kaggle, the Data Science Agent unlocks new frontiers in intelligent, self-directed discovery.
|
||||
|
||||
🧭 Example Guide - Customized dataset
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
🔧 **Set up RD-Agent Environment**
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
- Before you start, please make sure you have installed RD-Agent and configured the environment for RD-Agent correctly. If you want to know how to install and configure the RD-Agent, please refer to the `documentation <../installation_and_configuration.html>`_.
|
||||
|
||||
- 🔩 **Setting the Environment variables at .env file**
|
||||
|
||||
- Determine the path where the data will be stored and add it to the ``.env`` file.
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
dotenv set DS_LOCAL_DATA_PATH <your local directory>/ds_data
|
||||
dotenv set DS_SCEN rdagent.scenarios.data_science.scen.DataScienceScen
|
||||
|
||||
📥 **Prepare Customized datasets**
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
- A data science competition dataset usually consists of two parts: ``competition dataset`` and ``evaluation dataset``. (We provide `a sample <https://github.com/microsoft/RD-Agent/tree/main/rdagent/scenarios/data_science/example>`_ of a customized dataset named: `arf-12-hours-prediction-task as a reference`.)
|
||||
|
||||
- The ``competition dataset`` contains **training data**, **test data**, **description files**, **formatted submission files**, **data sampling codes**.
|
||||
|
||||
- The ``evaluation dataset`` contains **standard answer file**, **data checking codes**, and **Code for calculation of scores**.
|
||||
|
||||
- We use the ``arf-12-hours-prediction-task`` data as a sample to introduce the preparation workflow for the competition dataset.
|
||||
|
||||
- Create a ``ds_data/source_data/arf-12-hours-prediction-task`` folder, which will be used to store your raw dataset.
|
||||
|
||||
- The raw files for the competition ``arf-12-hours-prediction-task`` have two files: ``ARF_12h.csv`` and ``X.npz``.
|
||||
|
||||
- Create a ``ds_data/source_data/arf-12-hours-prediction-task/prepare.py`` file that splits your raw data into **training data**, **test data**, **formatted submission file**, and **standard answer file**. (You will need to write a script based on your raw data.)
|
||||
|
||||
- The following shows the preprocessing code for the raw data of ``arf-12-hours-prediction-task``.
|
||||
|
||||
.. literalinclude:: ../../rdagent/scenarios/data_science/example/source_data/arf-12-hours-prediction-task/prepare.py
|
||||
:language: python
|
||||
:caption: ds_data/source_data/arf-12-hours-prediction-task/prepare.py
|
||||
:linenos:
|
||||
|
||||
- At the end of program execution, the ``ds_data`` folder structure will look like this:
|
||||
|
||||
.. code-block:: text
|
||||
|
||||
ds_data
|
||||
├── arf-12-hours-prediction-task
|
||||
│ ├── train
|
||||
│ │ ├── ARF_12h.csv
|
||||
│ │ └── X.npz
|
||||
│ ├── test
|
||||
│ │ ├── ARF_12h.csv
|
||||
│ │ └── X.npz
|
||||
│ └── sample_submission.csv
|
||||
├── eval
|
||||
│ └── arf-12-hours-prediction-task
|
||||
│ └── submission_test.csv
|
||||
└── source_data
|
||||
└── arf-12-hours-prediction-task
|
||||
├── ARF_12h.csv
|
||||
├── prepare.py
|
||||
└── X.npz
|
||||
|
||||
- Create a ``ds_data/arf-12-hours-prediction-task/description.md`` file to describe your competition, Objective, dataset, and other information.
|
||||
|
||||
- The following shows the description file for ``arf-12-hours-prediction-task``
|
||||
|
||||
.. literalinclude:: ../../rdagent/scenarios/data_science/example/arf-12-hours-prediction-task/description.md
|
||||
:language: markdown
|
||||
:caption: ds_data/arf-12-hours-prediction-task/description.md
|
||||
:linenos:
|
||||
|
||||
- Create a ``ds_data/arf-12-hours-prediction-task/sample.py`` file to construct the debugging sample data.
|
||||
|
||||
- The following shows the script for constructing the debugging sample data based on the ``arf-12-hours-prediction-task`` dataset implementation.
|
||||
|
||||
.. literalinclude:: ../../rdagent/scenarios/data_science/example/arf-12-hours-prediction-task/sample.py
|
||||
:language: markdown
|
||||
:caption: ds_data/arf-12-hours-prediction-task/sample.py
|
||||
:linenos:
|
||||
|
||||
- Create a ``ds_data/eval/arf-12-hours-prediction-task/valid.py`` file, which is used to check the validity of the submission files to ensure that their formatting is consistent with the reference file.
|
||||
|
||||
- The following shows a script that checks the validity of a submission based on the ``arf-12-hours-prediction-task`` data.
|
||||
|
||||
.. literalinclude:: ../../rdagent/scenarios/data_science/example/eval/arf-12-hours-prediction-task/valid.py
|
||||
:language: markdown
|
||||
:caption: ds_data/eval/arf-12-hours-prediction-task/valid.py
|
||||
:linenos:
|
||||
|
||||
- Create a ``ds_data/eval/arf-12-hours-prediction-task/grade.py`` file, which is used to calculate the score based on the submission file and the **standard answer file**, and output the result in JSON format.
|
||||
|
||||
- The following shows a grading script based on the ``arf-12-hours-prediction-task`` data implementation.
|
||||
|
||||
.. literalinclude:: ../../rdagent/scenarios/data_science/example/eval/arf-12-hours-prediction-task/grade.py
|
||||
:language: markdown
|
||||
:caption: ds_data/eval/arf-12-hours-prediction-task/grade.py
|
||||
:linenos:
|
||||
|
||||
- At this point, you have created a complete dataset. The correct structure of the dataset should look like this.
|
||||
|
||||
.. code-block:: text
|
||||
|
||||
ds_data
|
||||
├── arf-12-hours-prediction-task
|
||||
│ ├── train
|
||||
│ │ ├── ARF_12h.csv
|
||||
│ │ └── X.npz
|
||||
│ ├── test
|
||||
│ │ ├── ARF_12h.csv
|
||||
│ │ └── X.npz
|
||||
│ ├── description.md
|
||||
│ ├── sample_submission.csv
|
||||
│ └── sample.py
|
||||
├── eval
|
||||
│ └── arf-12-hours-prediction-task
|
||||
│ ├── grade.py
|
||||
│ ├── submission_test.csv
|
||||
│ └── valid.py
|
||||
└── source_data
|
||||
└── arf-12-hours-prediction-task
|
||||
├── ARF_12h.csv
|
||||
├── prepare.py
|
||||
└── X.npz
|
||||
|
||||
- The above shows the complete dataset creation workflow, some of the files are not required, in practice you can customize the dataset according to your own needs.
|
||||
|
||||
- If we don't need the test set scores, then we can choose not to generate **formatted submission files** and **standard answer file** in the prepare code, and we don't need to write **data checking codes** and **Code for calculation of scores**.
|
||||
|
||||
- **Data sampling code** can also be created according to the actual need, if you do not provide **data sampling code**, RD-Agent will be handed over to the LLM sampling at runtime.
|
||||
|
||||
- In the default sampling method (``create_debug_data``), the default sampling ratio (parameter: ``min_frac``) is 1%, if 1% of the data is less than 5, then 5 data will be sampled (parameter: ``min_num``), you can adjust the sampling ratio by adjusting these two parameters.
|
||||
|
||||
- If you have customized data sampling code, you need to set ``DS_SAMPLE_DATA_BY_LLM`` to ``False`` (default is True) in the ``.env`` file before running, so that the program will use the customized sampling code when running, and you can just execute this line of code in the command line:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
dotenv set DS_SAMPLE_DATA_BY_LLM False
|
||||
|
||||
- In addition, we provide a data sampling method in `rdagent.scenarios.data_science.debug.data.create_debug_data <https://github.com/microsoft/RD-Agent/blob/main/rdagent/scenarios/data_science/debug/data.py#L605>`_, in this method, the default sampling ratio (parameter: ``min_frac``) is 1%, if 1% of the data is less than 5, then 5 data will be sampled (parameter: ``min_num``), you can use this method by the following two ways.
|
||||
|
||||
- You can set ``DS_SAMPLE_DATA_BY_LLM`` to ``False`` in the ``.env`` file so that when the program runs, it will use the sampling code provided by RD-Agent.
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
dotenv set DS_SAMPLE_DATA_BY_LLM False
|
||||
|
||||
- If you think that the parameters in the receipt sampling method provided by RD-Agent are not suitable, you can customize the parameters in the following command and run it, and set ``DS_SAMPLE_DATA_BY_LLM`` to ``False`` in the ``.env`` so that the program will use the sampling data you provided when running.
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
python rdagent/app/data_science/debug.py --dataset_path <dataset path> --competition <competiton_name> --min_frac <sampling ratio> --min_num <minimum number of sampling>
|
||||
dotenv set DS_SAMPLE_DATA_BY_LLM False
|
||||
|
||||
- If you don't need the scores from the test set and leave the data sampling to the LLM, or if you use the sampling method provided by the RD-Agent, you only need to prepare a minimal dataset. The structure of the simplest dataset should be as shown below.
|
||||
|
||||
.. code-block:: text
|
||||
|
||||
ds_data
|
||||
├── arf-12-hours-prediction-task
|
||||
│ ├── train
|
||||
│ │ ├── ARF_12h.csv
|
||||
│ │ └── X.npz
|
||||
│ ├── test
|
||||
│ │ ├── ARF_12h.csv
|
||||
│ │ └── X.npz
|
||||
│ └── description.md
|
||||
└── source_data
|
||||
└── arf-12-hours-prediction-task
|
||||
├── ARF_12h.csv
|
||||
├── prepare.py
|
||||
└── X.npz
|
||||
|
||||
- We have prepared a dataset based on the above description for your reference. You can download it with the following command.
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
wget https://github.com/SunsetWolf/rdagent_resource/releases/download/ds_data/arf-12-hours-prediction-task.zip
|
||||
|
||||
⚙️ **Set up Environment for Customized datasets**
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
dotenv set DS_SCEN rdagent.scenarios.data_science.scen.DataScienceScen
|
||||
dotenv set DS_LOCAL_DATA_PATH <your local directory>/ds_data
|
||||
dotenv set DS_CODER_ON_WHOLE_PIPELINE True
|
||||
|
||||
- 📘 More Environment Variables (Optional)
|
||||
|
||||
- If you want to see all the available environment variables, you can refer to the configuration file for Data Science scenarios:
|
||||
|
||||
.. literalinclude:: ../../rdagent/app/data_science/conf.py
|
||||
:language: python
|
||||
:linenos:
|
||||
|
||||
- These variables allow you to have finer-grained control in Data Science scenarios.
|
||||
|
||||
🚀 **Run the Application**
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
- 🌏 You can directly run the application by using the following command:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
rdagent data_science --competition <Competition ID>
|
||||
|
||||
- The following shows the command to run based on the ``arf-12-hours-prediction-task`` data
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
rdagent data_science --competition arf-12-hours-prediction-task
|
||||
|
||||
- More CLI Parameters for `rdagent data_science` command:
|
||||
|
||||
.. automodule:: rdagent.app.data_science.loop
|
||||
:members:
|
||||
:no-index:
|
||||
|
||||
- 📈 Visualize the R&D Process
|
||||
|
||||
- We provide a web UI to visualize the log. You just need to run:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
rdagent ui --port <custom port> --log-dir <your log folder like "log/"> --data_science True
|
||||
|
||||
- Then you can input the log path and visualize the R&D process.
|
||||
|
||||
- 🧪 Scoring the test results
|
||||
|
||||
- Finally, shutdown the program, and get the test set scores with this command.
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
dotenv run -- python rdagent/log/mle_summary.py grade <url_to_log>
|
||||
|
||||
Here, <url_to_log> refers to the parent directory of the log folder generated during the run.
|
||||
|
||||
🕹️ Kaggle Agent
|
||||
~~~~~~~~~~~~~~~~
|
||||
|
||||
📖 Background
|
||||
^^^^^^^^^^^^^^
|
||||
|
||||
In the landscape of data science competitions, Kaggle serves as the ultimate arena where data enthusiasts harness the power of algorithms to tackle real-world challenges.
|
||||
The Kaggle Agent stands as a pivotal tool, empowering participants to seamlessly integrate cutting-edge models and datasets, transforming raw data into actionable insights.
|
||||
|
||||
By utilizing the **Kaggle Agent**, data scientists can craft innovative solutions that not only uncover hidden patterns but also drive significant advancements in predictive accuracy and model robustness.
|
||||
|
||||
🧭 Example Guide - Kaggle Dataset
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
🛠️ Preparing For The Competition
|
||||
""""""""""""""""""""""""""""""""""
|
||||
|
||||
- 🔨 **Configuring the Kaggle API**
|
||||
|
||||
- Register and login on the `Kaggle <https://www.kaggle.com/>`_ website.
|
||||
- Click on the avatar (usually in the top right corner of the page) -> ``Settings`` -> ``Create New Token``, A file called ``kaggle.json`` will be downloaded.
|
||||
- Move ``kaggle.json`` to ``~/.config/kaggle/``
|
||||
- Modify the permissions of the ``kaggle.json`` file.
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
chmod 600 ~/.config/kaggle/kaggle.json
|
||||
|
||||
- For more information about Kaggle API Settings, refer to the `Kaggle API <https://github.com/Kaggle/kaggle-api>`_.
|
||||
|
||||
- 🔩 **Setting the Environment variables at .env file**
|
||||
|
||||
- Determine the path where the data will be stored and add it to the ``.env`` file.
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
mkdir -p <your local directory>/ds_data
|
||||
dotenv set KG_LOCAL_DATA_PATH <your local directory>/ds_data
|
||||
|
||||
- 📘 More Environment Variables (Optional)
|
||||
|
||||
- If you want to see all the available environment variables, you can refer to the configuration file for Data Science scenarios:
|
||||
|
||||
.. literalinclude:: ../../rdagent/app/data_science/conf.py
|
||||
:language: python
|
||||
:linenos:
|
||||
|
||||
- These variables allow you to have finer-grained control in Data Science scenarios.
|
||||
|
||||
- 🗳️ **Join the competition**
|
||||
|
||||
- If your Kaggle API account has not joined a competition, you will need to join the competition before running the program.
|
||||
|
||||
- At the bottom of the competition details page, you can find the ``Join the competition`` button, click on it and select ``I Understand and Accept`` to join the competition.
|
||||
|
||||
- In the **Competition List Available** below, you can jump to the competition details page.
|
||||
|
||||
📥 Preparing Competition DataDataset && Set up RD-Agent Environment
|
||||
""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
|
||||
|
||||
- As a subset of data science, kaggle's dataset still follows the data science format. Based on this, the kaggle dataset can be divided into two categories depending on whether or not it is supported by the **MLE-Bench**.
|
||||
|
||||
- What is **MLE-Bench**?
|
||||
|
||||
- **MLE-Bench** is a comprehensive benchmark designed to evaluate the **machine learning engineering** capabilities of AI systems using real-world scenarios. The dataset includes multiple Kaggle competitions. Since Kaggle does not provide reserved test sets for these competitions, the benchmark includes preparation scripts for splitting publicly available training data into new training and test sets, and scoring scripts for each competition to accurately evaluate submission scores.
|
||||
|
||||
- I'm running a competition Is **MLE-Bench** supported?
|
||||
|
||||
- You can see all the competitions supported by **MLE-Bench** `here <https://github.com/openai/mle-bench/tree/main/mlebench/competitions>`_.
|
||||
|
||||
- Prepare datasets for **MLE-Bench** supported competitions.
|
||||
|
||||
- If you agree with the **MLE-Bench** standard, then you don't need to prepare the dataset, you just need to configure your ``.env`` file to automate the download of the dataset.
|
||||
|
||||
- Configure environment variables, add ``DS_IF_USING_MLE_DATA`` to environment variables, and set it to ``True``.
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
dotenv set DS_IF_USING_MLE_DATA True
|
||||
|
||||
- Configure environment variables, add ``DS_SAMPLE_DATA_BY_LLM`` to environment variables, and set it to ``True``.
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
dotenv set DS_SAMPLE_DATA_BY_LLM True
|
||||
|
||||
- Configure environment variables, add ``DS_SCEN`` to environment variables, and set it to ``rdagent.scenarios.data_science.scen.KaggleScen``.
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
dotenv set DS_SCEN rdagent.scenarios.data_science.scen.KaggleScen
|
||||
|
||||
- At this point, you are ready to start running your competition, which will automatically download the data, and the LLM will automatically extract the minimum dataset.
|
||||
|
||||
- After running the program the structure of the ds_data folder should look like this (Using the ``tabular-playground-series-dec-2021`` contest as an example).
|
||||
|
||||
.. code-block:: text
|
||||
|
||||
ds_data
|
||||
├── tabular-playground-series-dec-2021
|
||||
│ ├── description.md
|
||||
│ ├── sample_submission.csv
|
||||
│ ├── test.csv
|
||||
│ └── train.csv
|
||||
└── zip_files
|
||||
└── tabular-playground-series-dec-2021
|
||||
└── tabular-playground-series-dec-2021.zip
|
||||
|
||||
- The ``ds_data/zip_files`` folder contains a zip file of the raw competition data downloaded from kaggle website.
|
||||
|
||||
- At runtime, RD-Agent will automatically build the Docker image specified at `rdagent/scenarios/kaggle/docker/mle_bench_docker/Dockerfile <https://github.com/microsoft/RD-Agent/blob/main/rdagent/scenarios/kaggle/docker/mle_bench_docker/Dockerfile>`_. This image is responsible for downloading the required datasets and grading files for MLE-Bench.
|
||||
|
||||
Note: The first run may take longer than subsequent runs as the Docker image and data are being downloaded and set up for the first time.
|
||||
|
||||
- Prepare datasets for competitions that are not supported by **MLE-Bench**.
|
||||
|
||||
- As a subset of data science, we can follow the format and steps of data science dataset to prepare kaggle dataset. Below we will describe the workflow for preparing a kaggle dataset using the competition ``playground-series-s4e9`` as an example.
|
||||
|
||||
- Create a ``ds_data/source_data/playground-series-s4e9`` folder, which will be used to store your raw dataset.
|
||||
|
||||
- The raw files for the competition ``playground-series-s4e9`` have two files: ``train.csv``, ``test.csv``, ``sample_submission.csv``, and there are two ways to get the raw data:
|
||||
|
||||
- You can find the raw data required for the competition on the `official kaggle website <https://www.kaggle.com/competitions/playground-series-s4e9/data>`_.
|
||||
|
||||
- Or you can use the command line to download the raw data for the competition, the download command is as follows.
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
kaggle competitions download -c playground-series-s4e9
|
||||
|
||||
- Create a ``ds_data/source_data/playground-series-s4e9/prepare.py`` file that splits your raw data into **training data**, **test data**, **formatted submission file**, and **standard answer file**. (You will need to write a script based on your raw data.)
|
||||
|
||||
- The following shows the preprocessing code for the raw data of ``playground-series-s4e9``.
|
||||
|
||||
.. literalinclude:: ../../rdagent/scenarios/data_science/example/source_data/playground-series-s4e9/prepare.py
|
||||
:language: python
|
||||
:caption: ds_data/source_data/playground-series-s4e9/prepare.py
|
||||
:linenos:
|
||||
|
||||
- At the end of program execution, the ``ds_data`` folder structure will look like this:
|
||||
|
||||
.. code-block:: text
|
||||
|
||||
ds_data
|
||||
├── playground-series-s4e9
|
||||
│ ├── train.csv
|
||||
│ ├── test.csv
|
||||
│ └── sample_submission.csv
|
||||
├── eval
|
||||
│ └── playground-series-s4e9
|
||||
│ └── submission_test.csv
|
||||
└── source_data
|
||||
└── playground-series-s4e9
|
||||
├── prepare.py
|
||||
├── sample_submission.csv
|
||||
├── test.csv
|
||||
└── train.csv
|
||||
|
||||
- Create a ``ds_data/playground-series-s4e9/description.md`` file to describe your competition, dataset description, and other information. We can find the `competition description information <https://www.kaggle.com/competitions/playground-series-s4e9/overview>`_ and the `dataset description information <https://www.kaggle.com/competitions/playground-series-s4e9/data>`_ from the Kaggle website.
|
||||
|
||||
- The following shows the description file for ``playground-series-s4e9``
|
||||
|
||||
.. literalinclude:: ../../rdagent/scenarios/data_science/example/playground-series-s4e9/description.md
|
||||
:language: markdown
|
||||
:caption: ds_data/playground-series-s4e9/description.md
|
||||
:linenos:
|
||||
|
||||
- Create a ``ds_data/eval/playground-series-s4e9/valid.py`` file, which is used to check the validity of the submission files to ensure that their formatting is consistent with the reference file.
|
||||
|
||||
- The following shows a script that checks the validity of a submission based on the ``playground-series-s4e9`` data.
|
||||
|
||||
.. literalinclude:: ../../rdagent/scenarios/data_science/example/eval/playground-series-s4e9/valid.py
|
||||
:language: markdown
|
||||
:caption: ds_data/eval/playground-series-s4e9/valid.py
|
||||
:linenos:
|
||||
|
||||
- Create a ``ds_data/eval/playground-series-s4e9/grade.py`` file, which is used to calculate the score based on the submission file and the **standard answer file**, and output the result in JSON format.
|
||||
|
||||
- The following shows a grading script based on the ``playground-series-s4e9`` data implementation.
|
||||
|
||||
.. literalinclude:: ../../rdagent/scenarios/data_science/example/eval/playground-series-s4e9/grade.py
|
||||
:language: markdown
|
||||
:caption: ds_data/eval/playground-series-s4e9/grade.py
|
||||
:linenos:
|
||||
|
||||
- In this example we don't create a ``ds_data/eval/playground-series-s4e9/sample.py``, we use the sample method provided by RD-Agent by default.
|
||||
|
||||
- At this point, you have created a complete dataset. The correct structure of the dataset should look like this.
|
||||
|
||||
.. code-block:: text
|
||||
|
||||
ds_data
|
||||
├── playground-series-s4e9
|
||||
│ ├── train.csv
|
||||
│ ├── test.csv
|
||||
│ ├── description.md
|
||||
│ └── sample_submission.csv
|
||||
├── eval
|
||||
│ └── playground-series-s4e9
|
||||
│ ├── grade.py
|
||||
│ ├── submission_test.csv
|
||||
│ └── valid.py
|
||||
└── source_data
|
||||
└── playground-series-s4e9
|
||||
├── prepare.py
|
||||
├── sample_submission.csv
|
||||
├── test.csv
|
||||
└── train.csv
|
||||
|
||||
- We have prepared a dataset based on the above description for your reference. You can download it with the following command.
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
wget https://github.com/SunsetWolf/rdagent_resource/releases/download/ds_data/playground-series-s4e9.zip
|
||||
|
||||
- Next, we need to configure the environment for the ``playground-series-s4e9`` contest. You can do this by executing the following command at the command line.
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
dotenv set DS_IF_USING_MLE_DATA False
|
||||
dotenv set DS_SAMPLE_DATA_BY_LLM False
|
||||
dotenv set DS_SCEN rdagent.scenarios.data_science.scen.KaggleScen
|
||||
|
||||
🚀 **Run the Application**
|
||||
""""""""""""""""""""""""""""""""""""
|
||||
|
||||
- 🌏 You can directly run the application by using the following command:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
rdagent data_science --competition <Competition ID>
|
||||
|
||||
- The following shows the command to run based on the ``playground-series-s4e9`` data
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
rdagent data_science --competition playground-series-s4e9
|
||||
|
||||
- More CLI Parameters for `rdagent data_science` command:
|
||||
|
||||
.. automodule:: rdagent.app.data_science.loop
|
||||
:members:
|
||||
:no-index:
|
||||
|
||||
- 📈 Visualize the R&D Process
|
||||
|
||||
- We provide a web UI to visualize the log. You just need to run:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
rdagent ui --port <custom port> --log-dir <your log folder like "log/"> --data_science True
|
||||
|
||||
- Then you can input the log path and visualize the R&D process.
|
||||
|
||||
- 🧪 Scoring the test results
|
||||
|
||||
- Finally, shutdown the program, and get the test set scores with this command.
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
dotenv run -- python rdagent/log/mle_summary.py grade <url_to_log>
|
||||
|
||||
- If you have configured the full output in ``ds_data/eval/playground-series-s4e9/grade.py``, or if you are running a competition that receives **MLE-Bench** support, you can also summarize the scores by running the following command.
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
rdagent grade_summary --log-folder=<url_to_log>
|
||||
|
||||
Here, <url_to_log> refers to the parent directory of the log folder generated during the run.
|
||||
@@ -0,0 +1,26 @@
|
||||
.. _finetune_agent:
|
||||
|
||||
================================
|
||||
FT-Agent for LLM Fine-Tuning
|
||||
================================
|
||||
|
||||
FT-Agent is the RD-Agent scenario for autonomous LLM fine-tuning, introduced in
|
||||
the ICML 2026 paper `FT-Dojo: Towards Autonomous LLM Fine-Tuning with Language
|
||||
Agents <https://arxiv.org/abs/2603.01712>`_.
|
||||
|
||||
The scenario automates benchmark-driven data processing, LLaMA-Factory training
|
||||
configuration, fail-fast validation, OpenCompass evaluation, and feedback-based
|
||||
iteration.
|
||||
|
||||
The full user guide is maintained in the repository:
|
||||
|
||||
`rdagent/app/finetune/llm/README.md <https://github.com/microsoft/RD-Agent/blob/main/rdagent/app/finetune/llm/README.md>`_
|
||||
|
||||
Minimal command after configuring the required ``FT_*`` settings:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
rdagent llm_finetune --base-model Qwen/Qwen2.5-7B-Instruct
|
||||
|
||||
Please read the full guide before running this scenario. A first run can download
|
||||
large dataset/model assets and consume LLM API calls and GPU hours.
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 152 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 12 KiB |
@@ -0,0 +1,150 @@
|
||||
.. _model_agent_fin:
|
||||
|
||||
=======================
|
||||
Finance Model Agent
|
||||
=======================
|
||||
|
||||
**🤖 Automated Quantitative Trading & Iterative Model Evolution**
|
||||
------------------------------------------------------------------------------------------
|
||||
|
||||
📖 Background
|
||||
~~~~~~~~~~~~~~
|
||||
In the realm of quantitative finance, both factor discovery and model development play crucial roles in driving performance.
|
||||
While much attention is often given to the discovery of new financial factors, the **models** that leverage these factors are equally important.
|
||||
The effectiveness of a quantitative strategy depends not only on the factors used but also on how well these factors are integrated into robust, predictive models.
|
||||
|
||||
However, the process of developing and optimizing these models can be labor-intensive and complex, requiring continuous refinement and adaptation to ever-changing market conditions.
|
||||
And this is where the **Finance Model Agent** steps in.
|
||||
|
||||
|
||||
🎥 `Demo <https://rdagent.azurewebsites.net/model_loop>`_
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. raw:: html
|
||||
|
||||
<div style="display: flex; justify-content: center; align-items: center;">
|
||||
<video width="600" controls>
|
||||
<source src="https://rdagent.azurewebsites.net/media/d85e8cab1da1cd3501d69ce837452f53a971a24911eae7bfa9237137.mp4" type="video/mp4">
|
||||
Your browser does not support the video tag.
|
||||
</video>
|
||||
</div>
|
||||
|
||||
|
||||
🌟 Introduction
|
||||
~~~~~~~~~~~~~~~~
|
||||
|
||||
In this scenario, our automated system proposes hypothesis, constructs model, implements code, conducts back-testing, and utilizes feedback in a continuous, iterative process.
|
||||
|
||||
The goal is to automatically optimize performance metrics within the Qlib library, ultimately discovering the most efficient code through autonomous research and development.
|
||||
|
||||
Here's an enhanced outline of the steps:
|
||||
|
||||
**Step 1 : Hypothesis Generation 🔍**
|
||||
|
||||
- Generate and propose initial hypotheses based on previous experiment analysis and domain expertise, with thorough reasoning and financial justification.
|
||||
|
||||
**Step 2 : Model Creation ✨**
|
||||
|
||||
- Transform the hypothesis into a task.
|
||||
- Develop, define, and implement a quantitative model, including its name, description, and formulation.
|
||||
|
||||
**Step 3 : Model Implementation 👨💻**
|
||||
|
||||
- Implement the model code based on the detailed description.
|
||||
- Evolve the model iteratively as a developer would, ensuring accuracy and efficiency.
|
||||
|
||||
**Step 4 : Backtesting with Qlib 📉**
|
||||
|
||||
- Conduct backtesting using the newly developed model and 20 factors extracted from Alpha158 in Qlib.
|
||||
- Evaluate the model's effectiveness and performance.
|
||||
|
||||
+----------------+------------+------------------------+----------------------------------------------------+
|
||||
| Dataset | Model | Factors | Data Split |
|
||||
+================+============+========================+====================================================+
|
||||
| CSI300 | RDAgent-dev| 20 factors (Alpha158) | +-----------+--------------------------+ |
|
||||
| | | | | Train | 2008-01-01 to 2014-12-31 | |
|
||||
| | | | +-----------+--------------------------+ |
|
||||
| | | | | Valid | 2015-01-01 to 2016-12-31 | |
|
||||
| | | | +-----------+--------------------------+ |
|
||||
| | | | | Test | 2017-01-01 to 2020-08-01 | |
|
||||
| | | | +-----------+--------------------------+ |
|
||||
+----------------+------------+------------------------+----------------------------------------------------+
|
||||
|
||||
**Step 5 : Feedback Analysis 🔍**
|
||||
|
||||
- Analyze backtest results to assess performance.
|
||||
- Incorporate feedback to refine hypotheses and improve the model.
|
||||
|
||||
**Step 6 :Hypothesis Refinement ♻️**
|
||||
|
||||
- Refine hypotheses based on feedback from backtesting.
|
||||
- Repeat the process to continuously improve the model.
|
||||
|
||||
⚡ Quick Start
|
||||
~~~~~~~~~~~~~~~~~
|
||||
|
||||
Please refer to the installation part in :doc:`../installation_and_configuration` to prepare your system dependency.
|
||||
|
||||
You can try our demo by running the following command:
|
||||
|
||||
- 🐍 Create a Conda Environment
|
||||
|
||||
- Create a new conda environment with Python (3.10 and 3.11 are well tested in our CI):
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
conda create -n rdagent python=3.10
|
||||
|
||||
- Activate the environment:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
conda activate rdagent
|
||||
|
||||
- 📦 Install the RDAgent
|
||||
|
||||
- You can install the RDAgent package from PyPI:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
pip install rdagent
|
||||
|
||||
- 🚀 Run the Application
|
||||
|
||||
- You can directly run the application by using the following command:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
rdagent fin_model
|
||||
|
||||
🛠️ Usage of modules
|
||||
~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. _Env Config:
|
||||
|
||||
- **Env Config**
|
||||
|
||||
The following environment variables can be set in the `.env` file to customize the application's behavior:
|
||||
|
||||
.. autopydantic_settings:: rdagent.app.qlib_rd_loop.conf.ModelBasePropSetting
|
||||
:settings-show-field-summary: False
|
||||
:exclude-members: Config
|
||||
|
||||
- **Qlib Config**
|
||||
- The `config.yaml` file located in the `model_template` folder contains the relevant configurations for running the developed model in Qlib. The default settings include key information such as:
|
||||
- **market**: Specifies the market, which is set to `csi300`.
|
||||
- **fields_group**: Defines the fields group, with the value `feature`.
|
||||
- **col_list**: A list of columns used, including various indicators such as `RESI5`, `WVMA5`, `RSQR5`, and others.
|
||||
- **start_time**: The start date for the data, set to `2008-01-01`.
|
||||
- **end_time**: The end date for the data, set to `2020-08-01`.
|
||||
- **fit_start_time**: The start date for fitting the model, set to `2008-01-01`.
|
||||
- **fit_end_time**: The end date for fitting the model, set to `2014-12-31`.
|
||||
|
||||
- The default hyperparameters used in the configuration are as follows:
|
||||
- **n_epochs**: The number of epochs, set to `100`.
|
||||
- **lr**: The learning rate, set to `1e-3`.
|
||||
- **early_stop**: The early stopping criterion, set to `10`.
|
||||
- **batch_size**: The batch size, set to `2000`.
|
||||
- **metric**: The evaluation metric, set to `loss`.
|
||||
- **loss**: The loss function, set to `mse`.
|
||||
- **n_jobs**: The number of parallel jobs, set to `20`.
|
||||
@@ -0,0 +1,99 @@
|
||||
.. _model_copilot_general:
|
||||
|
||||
======================
|
||||
General Model Copilot
|
||||
======================
|
||||
|
||||
**🤖 Automated Model Research & Development Co-Pilot**
|
||||
--------------------------------------------------------
|
||||
|
||||
📖 Background
|
||||
~~~~~~~~~~~~~~
|
||||
In the fast-paced field of artificial intelligence, the number of academic papers published each year is skyrocketing.
|
||||
These papers introduce new models, techniques, and approaches that can significantly advance the state of the art.
|
||||
However, reproducing and implementing these models can be a daunting task, requiring substantial time and expertise.
|
||||
Researchers often face challenges in extracting the essential details from these papers and converting them into functional code.
|
||||
And this is where the **General Model Copilot** steps in.
|
||||
|
||||
🎥 `Demo <https://rdagent.azurewebsites.net/report_model>`_
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. raw:: html
|
||||
|
||||
<div style="display: flex; justify-content: center; align-items: center;">
|
||||
<video width="600" controls>
|
||||
<source src="https://rdagent.azurewebsites.net/media/b35f904765b05099b0fcddbebe041a04f4d7bde239657e5fc24bf0cc.mp4" type="video/mp4">
|
||||
Your browser does not support the video tag.
|
||||
</video>
|
||||
</div>
|
||||
|
||||
🌟 Introduction
|
||||
~~~~~~~~~~~~~~~~
|
||||
In this scenario, our automated system proposes hypotheses, constructs models, implements code, performs back-testing, and uses feedback to iterate continuously. The system aims to automatically optimize performance metrics from the Qlib library, finding the best code through autonomous research and development.
|
||||
|
||||
Model R&D CoPilot Scenario
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
**Overview**
|
||||
|
||||
This demo automates the extraction and iterative development of models from academic papers, ensuring functionality and correctness. This scenario automates the development of PyTorch models by reading academic papers or other sources. It supports various data types, including tabular, time-series, and graph data. The primary workflow involves two main components: the Reader and the Coder.
|
||||
|
||||
**Workflow Components**
|
||||
|
||||
1. **Reader**
|
||||
- Parses and extracts relevant model information from academic papers or sources, including architectures, parameters, and implementation details.
|
||||
- Uses Large Language Models to convert content into a structured format for the Coder.
|
||||
|
||||
2. **Evolving Coder**
|
||||
- Translates structured information from the Reader into executable PyTorch code.
|
||||
- Utilizes an evolving coding mechanism to ensure correct tensor shapes, verified with sample input tensors.
|
||||
- Iteratively refines the code to align with source material specifications.
|
||||
|
||||
**Supported Data Types**
|
||||
|
||||
- **Tabular Data:** Structured data with rows and columns, such as spreadsheets or databases.
|
||||
- **Time-Series Data:** Sequential data points indexed in time order, useful for forecasting and temporal pattern recognition.
|
||||
- **Graph Data:** Data structured as nodes and edges, suitable for network analysis and relational tasks.
|
||||
|
||||
⚡ Quick Start
|
||||
~~~~~~~~~~~~~~~~~
|
||||
|
||||
Please refer to the installation part in :doc:`../installation_and_configuration` to prepare your system dependency.
|
||||
|
||||
You can try our demo by running the following command:
|
||||
|
||||
- 🐍 Create a Conda Environment
|
||||
|
||||
- Create a new conda environment with Python (3.10 and 3.11 are well tested in our CI):
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
conda create -n rdagent python=3.10
|
||||
|
||||
- Activate the environment:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
conda activate rdagent
|
||||
|
||||
- 📦 Install the RDAgent
|
||||
|
||||
- You can install the RDAgent package from PyPI:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
pip install rdagent
|
||||
|
||||
|
||||
- 🚀 Run the Application
|
||||
|
||||
- Prepare relevant files (in pdf format) by uploading papers to the directory below and copy the path as report_file_path.
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
rdagent/scenarios/general_model
|
||||
|
||||
- Run the following command in your terminal within the same virtual environment:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
rdagent general_model --report-file-path=<path_to_pdf_file>
|
||||
@@ -0,0 +1,113 @@
|
||||
.. _quant_agent_fin:
|
||||
|
||||
=====================
|
||||
Finance Quant Agent
|
||||
=====================
|
||||
|
||||
|
||||
**🥇The First Data-Centric Quant Multi-Agent Framework RD-Agent(Q)**
|
||||
---------------------------------------------------------------------
|
||||
|
||||
R&D-Agent for Quantitative Finance, in short **RD-Agent(Q)**, is the first data-centric, multi-agent framework designed to automate the full-stack research and development of quantitative strategies via coordinated factor-model co-optimization.
|
||||
|
||||
You can learn more details about **RD-Agent(Q)** through the `paper <https://arxiv.org/abs/2505.15155>`_.
|
||||
|
||||
⚡ Quick Start
|
||||
~~~~~~~~~~~~~~~~~
|
||||
|
||||
Before you start, please make sure you have installed RD-Agent and configured the environment for RD-Agent correctly. If you want to know how to install and configure the RD-Agent, please refer to the `documentation <../installation_and_configuration.html>`_.
|
||||
|
||||
Then, you can run the framework by running the following command:
|
||||
|
||||
- 🐍 Create a Conda Environment
|
||||
|
||||
- Create a new conda environment with Python (3.10 and 3.11 are well tested in our CI):
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
conda create -n rdagent python=3.10
|
||||
|
||||
- Activate the environment:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
conda activate rdagent
|
||||
|
||||
- 📦 Install the RDAgent
|
||||
|
||||
- You can install the RDAgent package from PyPI:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
pip install rdagent
|
||||
|
||||
- 🚀 Run the Application
|
||||
|
||||
- You can directly run the application by using the following command:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
rdagent fin_quant
|
||||
|
||||
|
||||
🛠️ Usage of modules
|
||||
~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. _Env Config:
|
||||
|
||||
- **Env Config**
|
||||
|
||||
The following environment variables can be set in the `.env` file to customize the application's behavior:
|
||||
|
||||
.. autopydantic_settings:: rdagent.app.qlib_rd_loop.conf.QuantBasePropSetting
|
||||
:settings-show-field-summary: False
|
||||
:exclude-members: Config
|
||||
|
||||
.. autopydantic_settings:: rdagent.components.coder.factor_coder.config.FactorCoSTEERSettings
|
||||
:settings-show-field-summary: False
|
||||
:members: coder_use_cache, data_folder, data_folder_debug, file_based_execution_timeout, select_method, max_loop, knowledge_base_path, new_knowledge_base_path
|
||||
:exclude-members: Config, fail_task_trial_limit, v1_query_former_trace_limit, v1_query_similar_success_limit, v2_query_component_limit, v2_query_error_limit, v2_query_former_trace_limit, v2_error_summary, v2_knowledge_sampler
|
||||
:no-index:
|
||||
|
||||
- **Qlib Configuration**
|
||||
- The `.yaml` files in both the `model_template` and `factor_template` directories contain some configurations for running the corresponding models or factors within the Qlib framework. Below is an overview of their contents and roles:
|
||||
- **General Settings**:
|
||||
- **provider_uri**: Specifies the local Qlib data path, set to `~/.qlib/qlib_data/cn_data`.
|
||||
- **market**: Configured to `csi300`, representing the CSI 300 index constituents.
|
||||
- **benchmark**: Set to `SH000300`, used for backtesting evaluation.
|
||||
|
||||
- **Data Handling**:
|
||||
- **start_time** and **end_time**: Define the full data range, from `2008-01-01` to `2022-08-01`.
|
||||
- **fit_start_time**: The start date for fitting the model, set to `2008-01-01`.
|
||||
- **fit_end_time**: The end date for fitting the model, set to `2014-12-31`.
|
||||
- **features and labels**: Generated via a nested data loader combining `Alpha158DL` (for engineered features such as `RESI5`, `WVMA5`, `RSQR5`, `KLEN`, etc.) and a `StaticDataLoader` that loads precomputed factor files (`combined_factors_df.parquet`).
|
||||
- **normalization**: The pipeline includes `RobustZScoreNorm` (with clipping) and `Fillna` for inference, and `DropnaLabel` with `CSZScoreNorm` for training.
|
||||
|
||||
- **Training Configuration**:
|
||||
- **Model**: Uses `GeneralPTNN`, a PyTorch-based neural network model.
|
||||
- **Dataset Splits**:
|
||||
- **train**: `2008-01-01` to `2014-12-31`
|
||||
- **valid**: `2015-01-01` to `2016-12-31`
|
||||
- **test**: `2017-01-01` to `2020-08-01`
|
||||
|
||||
- **Default Hyperparameters** (can be overridden by command-line arguments):
|
||||
- **n_epochs**: `100`
|
||||
- **lr**: `2e-4`
|
||||
- **early_stop**: `10`
|
||||
- **batch_size**: `256`
|
||||
- **weight_decay**: `0.0`
|
||||
- **metric**: `loss`
|
||||
- **loss**: `mse`
|
||||
- **n_jobs**: `20`
|
||||
- **GPU**: `0` (uses GPU 0 if available)
|
||||
|
||||
- **Backtesting and Evaluation**:
|
||||
- **strategy**: `TopkDropoutStrategy`, which selects the top 50 stocks and randomly drops 5 to introduce exploration.
|
||||
- **backtest period**: `2017-01-01` to `2020-08-01`
|
||||
- **initial capital**: `100,000,000`
|
||||
- **cost configuration**: Includes open/close costs, minimum transaction costs, and slippage control.
|
||||
|
||||
- **Recording and Analysis**:
|
||||
- **SignalRecord**: Logs predicted signals.
|
||||
- **SigAnaRecord**: Performs signal analysis without long-short separation.
|
||||
- **PortAnaRecord**: Conducts portfolio analysis using the configured strategy and backtest settings.
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
==============
|
||||
User Interface
|
||||
==============
|
||||
|
||||
|
||||
Introduction
|
||||
============
|
||||
|
||||
RD-Agent will generate some logs during the R&D process. These logs are very useful for debugging and understanding the R&D process. However, just viewing the terminal log is not intuitive enough. RD-Agent provides a web app as UI to visualize the R&D process. You can easily view the R&D process and understand the R&D process better.
|
||||
|
||||
A Quick Demo
|
||||
============
|
||||
|
||||
Start Web App
|
||||
-------------
|
||||
|
||||
In `RD-Agent/` folder, run:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
rdagent ui --port <port> --log-dir <log_dir like "log/"> [--debug]
|
||||
|
||||
This will start a web app on `http://localhost:<port>`.
|
||||
|
||||
**NOTE**: The log_dir parameter is not required. You can manually enter the log_path in the web app. If you set the log_dir parameter, you can easily select a different log_path in the web app.
|
||||
|
||||
--debug is optional, it will show a "Single Step Run" button in sidebar and saved objects info in the web app.
|
||||
|
||||
Use Web App
|
||||
-----------
|
||||
|
||||
1. Open the sidebar.
|
||||
|
||||
.. TODO: update these
|
||||
|
||||
2. Select the scenario you want to show. There are some pre-defined scenarios:
|
||||
- Qlib Model
|
||||
- Qlib Factor
|
||||
- Data Mining
|
||||
- Model from Paper
|
||||
- Kaggle
|
||||
|
||||
3. Click the `Config⚙️` button and input the log path (if you set the log_dir parameter, you can select a log_path in the dropdown list).
|
||||
|
||||
4. Click the buttons below Config⚙️ to show the scenario execution process. Buttons are:
|
||||
- All Loops: Show complete scenario execution process.
|
||||
- Next Loop: Show one success **R&D Loop**.
|
||||
- One Evolving: Show one **evolving** step of **development** part.
|
||||
- refresh logs: clear shown logs.
|
||||
+130
@@ -0,0 +1,130 @@
|
||||
[build-system]
|
||||
build-backend = "setuptools.build_meta"
|
||||
requires = [
|
||||
"setuptools",
|
||||
"setuptools-scm",
|
||||
]
|
||||
|
||||
[project]
|
||||
authors = [
|
||||
{email = "xuyang1@microsoft.com", name = "MSRA-MIIC"},
|
||||
]
|
||||
classifiers = [
|
||||
"Development Status :: 3 - Alpha",
|
||||
"License :: OSI Approved :: MIT License",
|
||||
"Operating System :: OS Independent",
|
||||
"Programming Language :: Python :: 3.10",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
]
|
||||
description = "Research & Development Agent"
|
||||
dynamic = [
|
||||
"dependencies",
|
||||
"optional-dependencies",
|
||||
"version",
|
||||
]
|
||||
keywords = [
|
||||
"Autonomous Agents",
|
||||
"Large Language Models",
|
||||
"Research and Development",
|
||||
]
|
||||
name = "rdagent"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.10"
|
||||
|
||||
[project.scripts]
|
||||
rdagent = "rdagent.app.cli:app"
|
||||
|
||||
[project.urls]
|
||||
homepage = "https://github.com/microsoft/RD-Agent/"
|
||||
issue = "https://github.com/microsoft/RD-Agent/issues"
|
||||
|
||||
[tool.coverage.report]
|
||||
fail_under = 80
|
||||
|
||||
[tool.coverage.run]
|
||||
source = [
|
||||
"rdagent",
|
||||
]
|
||||
|
||||
[tool.isort]
|
||||
color_output = true
|
||||
profile = "black"
|
||||
|
||||
[tool.mypy]
|
||||
check_untyped_defs = true
|
||||
disallow_any_unimported = true
|
||||
disallow_untyped_defs = true
|
||||
enable_error_code = [
|
||||
"ignore-without-code",
|
||||
]
|
||||
explicit_package_bases = true
|
||||
warn_return_any = true
|
||||
warn_unused_ignores = true
|
||||
|
||||
[[tool.mypy.overrides]]
|
||||
ignore_missing_imports = true
|
||||
module = "llama"
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
addopts = "-l -s --durations=0"
|
||||
log_cli = true
|
||||
log_cli_level = "info"
|
||||
log_date_format = "%Y-%m-%d %H:%M:%S"
|
||||
log_format = "%(asctime)s %(levelname)s %(message)s"
|
||||
markers = [
|
||||
"offline: tests that do not require external API calls",
|
||||
]
|
||||
minversion = "6.0"
|
||||
norecursedirs = [
|
||||
"workspace",
|
||||
]
|
||||
|
||||
[tool.ruff]
|
||||
fix = true
|
||||
line-length = 120
|
||||
src = ["rdagent"]
|
||||
|
||||
[tool.ruff.lint]
|
||||
ignore = [
|
||||
# https://docs.astral.sh/ruff/rules/#pydocstyle-d
|
||||
"ANN401",
|
||||
"D",
|
||||
"ERA001",
|
||||
"EXE002",
|
||||
"FIX",
|
||||
"INP001",
|
||||
"PGH",
|
||||
"PLR0913",
|
||||
"S101",
|
||||
"S301",
|
||||
"T20",
|
||||
"TC003",
|
||||
"TD",
|
||||
]
|
||||
select = ["ALL"]
|
||||
|
||||
[tool.ruff.lint.per-file-ignores]
|
||||
"docs/conf.py" = ["INP001"]
|
||||
"test/*" = ["S101"]
|
||||
|
||||
[tool.setuptools]
|
||||
packages = ["rdagent"]
|
||||
|
||||
[tool.setuptools.dynamic]
|
||||
dependencies = {file = ["requirements.txt"]}
|
||||
|
||||
[tool.setuptools.dynamic.optional-dependencies]
|
||||
docs = {file = ["requirements/docs.txt"]}
|
||||
lint = {file = ["requirements/lint.txt"]}
|
||||
package = {file = ["requirements/package.txt"]}
|
||||
test = {file = ["requirements/test.txt"]}
|
||||
torch = {file = ["requirements/torch.txt"]} # some agent algorithms need torch. pip install rdagent[torch]
|
||||
|
||||
[tool.setuptools_scm]
|
||||
local_scheme = "no-local-version"
|
||||
version_scheme = "guess-next-dev"
|
||||
|
||||
[tool.tomlsort]
|
||||
all = true
|
||||
in_place = true
|
||||
trailing_comma_inline_array = true
|
||||
@@ -0,0 +1,38 @@
|
||||
# CI 检查
|
||||
|
||||
`.github/workflows/ci.yml`配置了提交时自动运行`Makefile`: 91~103行的命令，可以在这调整执行的命令
|
||||
|
||||
在`.env`中设置`USE_CHAT_CACHE=True`可以让第二次修复快一些
|
||||
|
||||
# Rules
|
||||
|
||||
`pyproject.toml`中配置全局屏蔽的规则
|
||||
- ruff: `[tool.ruff.lint].ignore`
|
||||
- mypy: `[tool.mypy]`
|
||||
|
||||
## ruff rules
|
||||
ruff rules 比较好修改, 大多可以自动修复
|
||||
|
||||
对于一些规则可以在代码中添加注释来局部屏蔽, 例如添加 `# noqa E234,ANN001`
|
||||
遇到的不好修改的规则:
|
||||
- 捕获异常时应该处理每一种异常，不应该统一当作`Exception`处理
|
||||
- `subprogress()` 调用命令应该先判断命令是否安全
|
||||
- ...
|
||||
|
||||
规则列表: [ruff rules](https://docs.astral.sh/ruff/rules/)
|
||||
|
||||
## mypy rules
|
||||
|
||||
Mypy检查Python中类型标注, 常遇到需要修改结构/同时修改其他文件的情况, 自动修复效果不好
|
||||
|
||||
局部屏蔽: `# type: ignore`
|
||||
|
||||
规则列表: [mypy rules](https://mypy.readthedocs.io/en/stable/error_code_list.html)
|
||||
|
||||
# Optimization (Maybe)
|
||||
|
||||
- 添加指定文件夹检查的功能
|
||||
- 增加一个修改选项: 调用`vim`, 用户直接修改此部分代码
|
||||
- 显示时把`Original Code`部分去掉, 直接在输出的表示修改的diff部分用`^^^^^^`在代码行下标注出错误位置，这样能更直观地观察错误修复情况
|
||||
- 当前为线性执行完所有修复后交给用户检查, 可修改成 后台多线程 / 进程处理修复的任务, 终端实时展示处理完的修复让用户检查
|
||||
- ...
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "rdagent",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"name": "python",
|
||||
"version": "3.10.0"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
generate_lint_command_template: |
|
||||
Please generate a command to lint or format a {language} repository.
|
||||
Here are some information about different linting tools ```{linting_tools}```
|
||||
linting_system_prompt_template: |
|
||||
You are a software engineer. You can write code to a high standard and are adept at solving {language} linting problems.
|
||||
session_manual_template: |
|
||||
There are some problems with the code you provided, please modify the code again according to the instruction and return the errors list you modified.
|
||||
|
||||
Instruction:
|
||||
{operation}
|
||||
|
||||
Your response format should be like this:
|
||||
|
||||
```python
|
||||
<modified code>
|
||||
```
|
||||
|
||||
```json
|
||||
{{
|
||||
"errors": ["<Line Number>:<Error Start Position> <Error Code>", ...]
|
||||
}}
|
||||
```
|
||||
session_normal_template: |
|
||||
Please modify this code snippet based on the lint info. Here is the code snippet:
|
||||
```Python
|
||||
{code}
|
||||
```
|
||||
|
||||
-----Lint info-----
|
||||
{lint_info}
|
||||
-------------------
|
||||
|
||||
The lint info contains one or more errors. Different errors are separated by blank lines. Each error follows this format:
|
||||
-----Lint info format-----
|
||||
<Line Number>:<Error Start Position> <Error Code> <Error Message>
|
||||
<Error Position (maybe multiple lines)>
|
||||
<Helpful Information (sometimes have)>
|
||||
--------------------------
|
||||
The error code is an abbreviation set by the checker for ease of describing the error. The error position includes the relevant code around the error, and the helpful information provides useful information or possible fix method.
|
||||
|
||||
Please simply reply the code after you fix all linting errors. You should be aware of the following:
|
||||
1. The indentation of the code should be consistent with the original code.
|
||||
2. You should just replace the code I provided you, which starts from line {start_line} to line {end_line}.
|
||||
3. You'll need to add line numbers to the modified code which starts from {start_lineno}.
|
||||
4. You don't need to add comments to explain your changes.
|
||||
Please wrap your code with following format:
|
||||
|
||||
```python
|
||||
<your code..>
|
||||
```
|
||||
session_start_template: |
|
||||
Please modify the Python code based on the lint info.
|
||||
Due to the length of the code, I will first tell you the entire code, and then each time I ask a question, I will extract a portion of the code and tell you the error information contained in this code segment.
|
||||
You need to fix the corresponding error in the code segment and return the code that can replace the corresponding code segment.
|
||||
|
||||
The Python code is from a complete Python project file. Each line of the code is annotated with a line number, separated from the original code by three characters ("<white space>|<white space>"). The vertical bars are aligned.
|
||||
Here is the complete code, please be prepared to fix it:
|
||||
```Python
|
||||
{code}
|
||||
```
|
||||
suffix2language_template: |
|
||||
Here are the files suffix in one code repo: {suffix}.
|
||||
Please tell me the programming language used in this repo and which language has linting-tools.
|
||||
Your response should follow this template:
|
||||
{{
|
||||
"languages": <languages list>,
|
||||
"languages_with_linting_tools": <languages with lingting tools list>
|
||||
}}
|
||||
user_get_files_contain_lint_commands_template: |
|
||||
You get a file list of a repository. Some files may contain linting rules or linting commands defined by repo authors.
|
||||
Here are the file list:
|
||||
```
|
||||
{file_list}
|
||||
```
|
||||
|
||||
Please find all files that may correspond to linting from it.
|
||||
Please respond with the following JSON template:
|
||||
{{
|
||||
"files": </path/to/file>,
|
||||
}}
|
||||
user_get_makefile_lint_commands_template: |
|
||||
You get a Makefile which contains some linting rules. Here are its content:
|
||||
```
|
||||
{file_text}
|
||||
```
|
||||
Please find executable commands about linting from it.
|
||||
Please respond with the following JSON template:
|
||||
{{
|
||||
"commands": ["python -m xxx --params"...],
|
||||
}}
|
||||
user_template_for_code_snippet: |
|
||||
Please modify the Python code based on the lint info.
|
||||
-----Python Code-----
|
||||
{code}
|
||||
---------------------
|
||||
|
||||
-----Lint info-----
|
||||
{lint_info}
|
||||
-------------------
|
||||
|
||||
The Python code is a snippet from a complete Python project file. Each line of the code is annotated with a line number, separated from the original code by three characters ("<white space>|<white space>"). The vertical bars are aligned.
|
||||
|
||||
The lint info contains one or more errors. Different errors are separated by blank lines. Each error follows this format:
|
||||
-----Lint info format-----
|
||||
<Line Number>:<Error Start Position> <Error Code> <Error Message>
|
||||
<Error Context (multiple lines)>
|
||||
<Helpful Information (last line)>
|
||||
--------------------------
|
||||
The error code is an abbreviation set by the checker for ease of describing the error. The error context includes the relevant code around the error, and the helpful information suggests possible fixes.
|
||||
|
||||
Please simply reply the code after you fix all linting errors.
|
||||
The code you return does not require line numbers, and should just replace the code I provided you, and does not require comments.
|
||||
Please wrap your code with following format:
|
||||
|
||||
```python
|
||||
<your code..>
|
||||
```
|
||||
@@ -0,0 +1,817 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime
|
||||
import json
|
||||
import re
|
||||
import shlex
|
||||
import subprocess
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass
|
||||
from difflib import ndiff
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
|
||||
import tree_sitter_python
|
||||
from rich import print
|
||||
from rich.panel import Panel
|
||||
from rich.progress import Progress, SpinnerColumn, TimeElapsedColumn
|
||||
from rich.prompt import Prompt
|
||||
from rich.rule import Rule
|
||||
from rich.syntax import Syntax
|
||||
from rich.table import Table
|
||||
from rich.text import Text
|
||||
from tree_sitter import Language, Node, Parser
|
||||
|
||||
from rdagent.core.evaluation import Evaluator
|
||||
from rdagent.core.evolving_agent import EvoAgent
|
||||
from rdagent.core.evolving_framework import (
|
||||
EvolvableSubjects,
|
||||
EvolvingStrategy,
|
||||
EvoStep,
|
||||
Feedback,
|
||||
Knowledge,
|
||||
)
|
||||
from rdagent.core.prompts import Prompts
|
||||
from rdagent.oai.llm_utils import APIBackend
|
||||
|
||||
py_parser = Parser(Language(tree_sitter_python.language()))
|
||||
CI_prompts = Prompts(file_path=Path(__file__).parent / "prompts.yaml")
|
||||
|
||||
|
||||
@dataclass
|
||||
class CIError:
|
||||
raw_str: str
|
||||
file_path: Path | str
|
||||
line: int
|
||||
column: int
|
||||
code: str
|
||||
msg: str
|
||||
hint: str
|
||||
checker: Literal["ruff", "mypy"]
|
||||
|
||||
def to_dict(self) -> dict[str, object]:
|
||||
return self.__dict__
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"{self.file_path}:{self.line}:{self.column}: {self.code} {self.msg}\n{self.hint}".strip()
|
||||
|
||||
|
||||
@dataclass
|
||||
class CIFeedback(Feedback):
|
||||
errors: dict[str, list[CIError]]
|
||||
|
||||
def statistics(self) -> dict[Literal["ruff", "mypy"], dict[str, int]]:
|
||||
error_counts = defaultdict(lambda: defaultdict(int))
|
||||
for file_errors in self.errors.values():
|
||||
for error in file_errors:
|
||||
error_counts[error.checker][error.code] += 1
|
||||
return error_counts
|
||||
|
||||
|
||||
@dataclass
|
||||
class FixRecord:
|
||||
skipped_errors: list[CIError]
|
||||
directly_fixed_errors: list[CIError]
|
||||
manually_fixed_errors: list[CIError]
|
||||
manual_instructions: dict[str, list[CIError]]
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"skipped_errors": [error.to_dict() for error in self.skipped_errors],
|
||||
"directly_fixed_errors": [error.to_dict() for error in self.directly_fixed_errors],
|
||||
"manually_fixed_errors": [error.to_dict() for error in self.manually_fixed_errors],
|
||||
"manual_instructions": {
|
||||
key: [error.to_dict() for error in errors] for key, errors in self.manual_instructions.items()
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class CodeFile:
|
||||
def __init__(self, path: Path | str) -> None:
|
||||
self.path = Path(path)
|
||||
self.load()
|
||||
|
||||
@classmethod
|
||||
def add_line_number(cls: CodeFile, code: list[str] | str, start: int = 1) -> list[str] | str:
|
||||
code_lines = code.split("\n") if isinstance(code, str) else code
|
||||
|
||||
lineno_width = len(str(start - 1 + len(code_lines)))
|
||||
code_with_lineno = []
|
||||
for i, code_line in enumerate(code_lines):
|
||||
code_with_lineno.append(f"{i+start: >{lineno_width}} | {code_line}")
|
||||
|
||||
return code_with_lineno if isinstance(code, list) else "\n".join(code_with_lineno)
|
||||
|
||||
@classmethod
|
||||
def remove_line_number(cls: CodeFile, code: list[str] | str) -> list[str] | str:
|
||||
code_lines = code.split("\n") if isinstance(code, str) else code
|
||||
|
||||
try:
|
||||
code_without_lineno = [re.split(r"\| ", code_line, maxsplit=1)[1] for code_line in code_lines]
|
||||
except IndexError:
|
||||
code_without_lineno = ["something went wrong when remove line numbers", *code_lines]
|
||||
|
||||
return code_without_lineno if isinstance(code, list) else "\n".join(code_without_lineno)
|
||||
|
||||
def load(self) -> None:
|
||||
code = self.path.read_text(encoding="utf-8")
|
||||
self.code_lines = code.split("\n")
|
||||
|
||||
# line numbers
|
||||
self.lineno = len(self.code_lines)
|
||||
self.lineno_width = len(str(self.lineno))
|
||||
self.code_lines_with_lineno = self.add_line_number(self.code_lines)
|
||||
|
||||
def get(
|
||||
self,
|
||||
start: int = 1,
|
||||
end: int | None = None,
|
||||
*,
|
||||
add_line_number: bool = False,
|
||||
return_list: bool = False,
|
||||
) -> list[str] | str:
|
||||
"""
|
||||
Retrieves a portion of the code lines.
|
||||
line number starts from 1, return codes in [start, end].
|
||||
|
||||
Args:
|
||||
start (int): The starting line number (inclusive). Defaults to 1.
|
||||
end (int | None): The ending line number (inclusive). Defaults to None, which means the last line.
|
||||
add_line_number (bool): Whether to include line numbers in the result. Defaults to False.
|
||||
return_list (bool): Whether to return the result as a list of lines
|
||||
or as a single string. Defaults to False.
|
||||
|
||||
Returns:
|
||||
list[str] | str: The code lines as a list of strings or as a
|
||||
single string, depending on the value of `return_list`.
|
||||
"""
|
||||
start -= 1
|
||||
if start < 0:
|
||||
start = 0
|
||||
end = self.lineno if end is None else end
|
||||
if end <= start:
|
||||
res = []
|
||||
res = self.code_lines_with_lineno[start:end] if add_line_number else self.code_lines[start:end]
|
||||
|
||||
return res if return_list else "\n".join(res)
|
||||
|
||||
def apply_changes(self, changes: list[tuple[int, int, str]]) -> None:
|
||||
"""
|
||||
Applies the given changes to the code lines.
|
||||
|
||||
Args:
|
||||
changes (List[Tuple[int, int, str]]): A list of tuples representing the changes to be applied.
|
||||
Each tuple contains the start line number, end line number, and the new code to be inserted.
|
||||
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
offset = 0
|
||||
for start, end, code in changes:
|
||||
# starts from 1 --> starts from 0
|
||||
adjusted_start = max(start - 1, 0)
|
||||
|
||||
new_code = code.split("\n")
|
||||
self.code_lines[adjusted_start + offset : end + offset] = new_code
|
||||
offset += len(new_code) - (end - adjusted_start)
|
||||
|
||||
self.path.write_text("\n".join(self.code_lines), encoding="utf-8")
|
||||
self.load()
|
||||
|
||||
def get_code_blocks(self, max_lines: int = 30) -> list[tuple[int, int]]:
|
||||
tree = py_parser.parse(bytes("\n".join(self.code_lines), "utf8"))
|
||||
|
||||
def get_blocks_in_node(node: Node, max_lines: int) -> list[tuple[int, int]]:
|
||||
if node.type == "assignment":
|
||||
return [(node.start_point.row, node.end_point.row + 1)]
|
||||
|
||||
blocks: list[tuple[int, int]] = []
|
||||
block: tuple[int, int] | None = None # [start, end), line number starts from 0
|
||||
|
||||
for child in node.children:
|
||||
if child.end_point.row + 1 - child.start_point.row > max_lines:
|
||||
if block is not None:
|
||||
blocks.append(block)
|
||||
block = None
|
||||
blocks.extend(get_blocks_in_node(child, max_lines))
|
||||
elif block is None:
|
||||
block = (child.start_point.row, child.end_point.row + 1)
|
||||
elif child.end_point.row + 1 - block[0] <= max_lines:
|
||||
block = (block[0], child.end_point.row + 1)
|
||||
else:
|
||||
blocks.append(block)
|
||||
block = (child.start_point.row, child.end_point.row + 1)
|
||||
|
||||
if block is not None:
|
||||
blocks.append(block)
|
||||
|
||||
return blocks
|
||||
|
||||
# change line number to start from 1 and [start, end) to [start, end]
|
||||
return [(a + 1, b) for a, b in get_blocks_in_node(tree.root_node, max_lines)]
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"{self.path}"
|
||||
|
||||
|
||||
class Repo(EvolvableSubjects):
|
||||
def __init__(self, project_path: Path | str, excludes: list[Path] | None = None, **kwargs: Any) -> None:
|
||||
if excludes is None:
|
||||
excludes = []
|
||||
self.params = kwargs
|
||||
self.project_path = Path(project_path)
|
||||
|
||||
excludes = [self.project_path / path for path in excludes]
|
||||
|
||||
git_ignored_output = subprocess.check_output(
|
||||
["/usr/bin/git", "status", "--ignored", "-s"], # noqa: S603
|
||||
cwd=str(self.project_path),
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
)
|
||||
git_ignored_files = [
|
||||
(self.project_path / Path(line[3:])).resolve()
|
||||
for line in git_ignored_output.split("\n")
|
||||
if line.startswith("!!")
|
||||
]
|
||||
|
||||
excludes.extend(git_ignored_files)
|
||||
|
||||
files = [
|
||||
file
|
||||
for file in self.project_path.glob("**/*")
|
||||
if file.is_file()
|
||||
and not any(str(file).startswith(str(path)) for path in excludes)
|
||||
and ".git/" not in str(file)
|
||||
and file.suffix == ".py"
|
||||
]
|
||||
self.files = {file: CodeFile(file) for file in files}
|
||||
|
||||
self.fix_records: dict[str, FixRecord] | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class RuffRule:
|
||||
"""
|
||||
Example:
|
||||
{
|
||||
"name": "missing-trailing-comma",
|
||||
"code": "COM812",
|
||||
"linter": "flake8-commas",
|
||||
"summary": "Trailing comma missing",
|
||||
"message_formats": [
|
||||
"Trailing comma missing"
|
||||
],
|
||||
"fix": "Fix is always available.",
|
||||
"explanation": "...",
|
||||
"preview": false
|
||||
}
|
||||
"""
|
||||
|
||||
name: str
|
||||
code: str
|
||||
linter: str
|
||||
summary: str
|
||||
message_formats: list[str]
|
||||
fix: str
|
||||
explanation: str
|
||||
preview: bool
|
||||
|
||||
|
||||
class RuffEvaluator(Evaluator):
|
||||
"""
|
||||
The error message are generated by command
|
||||
"""
|
||||
|
||||
def __init__(self, command: str | None = None) -> None:
|
||||
if command is None:
|
||||
self.command = "ruff check . --output-format full"
|
||||
else:
|
||||
self.command = command
|
||||
|
||||
@staticmethod
|
||||
def explain_rule(error_code: str) -> RuffRule:
|
||||
explain_command = f"ruff rule {error_code} --output-format json"
|
||||
try:
|
||||
out = subprocess.check_output(
|
||||
shlex.split(explain_command), # noqa: S603
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
)
|
||||
except subprocess.CalledProcessError as e:
|
||||
out = e.output
|
||||
|
||||
return RuffRule(**json.loads(out))
|
||||
|
||||
def evaluate(self, evo: Repo, **kwargs: dict) -> CIFeedback:
|
||||
"""Simply run ruff to get the feedbacks."""
|
||||
try:
|
||||
out = subprocess.check_output(
|
||||
shlex.split(self.command), # noqa: S603
|
||||
cwd=evo.project_path,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
)
|
||||
except subprocess.CalledProcessError as e:
|
||||
out = e.output
|
||||
|
||||
"""ruff output format:
|
||||
rdagent/cli.py:9:5: ANN201 Missing return type annotation for public function `main`
|
||||
|
|
||||
9 | def main(prompt=None):
|
||||
| ^^^^ ANN201
|
||||
10 | load_dotenv(verbose=True, override=True)
|
||||
11 | wm = WorkflowManager()
|
||||
|
|
||||
= help: Add return type annotation: `None`
|
||||
"""
|
||||
|
||||
# extract error info
|
||||
pattern = r"(([^\n]*):(\d+):(\d+): (\w+) ([^\n]*)\n(.*?))\n\n"
|
||||
matches = re.findall(pattern, out, re.DOTALL)
|
||||
|
||||
errors = defaultdict(list)
|
||||
|
||||
for match in matches:
|
||||
raw_str, file_path, line_number, column_number, error_code, error_message, error_hint = match
|
||||
|
||||
# TODO @bowen: filter these files when running the check command
|
||||
if evo.project_path / Path(file_path) not in evo.files:
|
||||
continue
|
||||
error = CIError(
|
||||
raw_str=raw_str,
|
||||
file_path=file_path,
|
||||
line=int(line_number),
|
||||
column=int(column_number),
|
||||
code=error_code,
|
||||
msg=error_message,
|
||||
hint=error_hint,
|
||||
checker="ruff",
|
||||
)
|
||||
|
||||
errors[file_path].append(error)
|
||||
|
||||
return CIFeedback(errors=errors)
|
||||
|
||||
|
||||
class MypyEvaluator(Evaluator):
|
||||
def __init__(self, command: str | None = None) -> None:
|
||||
if command is None:
|
||||
self.command = "mypy . --pretty --no-error-summary --show-column-numbers"
|
||||
else:
|
||||
self.command = command
|
||||
|
||||
def evaluate(self, evo: Repo, **kwargs: dict) -> CIFeedback:
|
||||
try:
|
||||
out = subprocess.check_output(
|
||||
shlex.split(self.command), # noqa: S603
|
||||
cwd=evo.project_path,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
)
|
||||
except subprocess.CalledProcessError as e:
|
||||
out = e.output
|
||||
|
||||
errors = defaultdict(list)
|
||||
|
||||
out = re.sub(r"([^\n]*?:\d+:\d+): error:", r"\n\1: error:", out)
|
||||
out += "\n"
|
||||
pattern = r"(([^\n]*?):(\d+):(\d+): error:(.*?)\s\[([\w-]*?)\]\s(.*?))\n\n"
|
||||
for match in re.findall(pattern, out, re.DOTALL):
|
||||
raw_str, file_path, line_number, column_number, error_message, error_code, error_hint = match
|
||||
error_message = error_message.strip().replace("\n", " ")
|
||||
if re.match(r".*[^\n]*?:\d+:\d+: note:.*", error_hint, flags=re.DOTALL) is not None:
|
||||
error_hint_position = re.split(
|
||||
pattern=r"[^\n]*?:\d+:\d+: note:",
|
||||
string=error_hint,
|
||||
maxsplit=1,
|
||||
flags=re.DOTALL,
|
||||
)[0]
|
||||
error_hint_help = re.findall(r"^.*?:\d+:\d+: note: (.*)$", error_hint, flags=re.MULTILINE)
|
||||
error_hint_help = "\n".join(error_hint_help)
|
||||
error_hint = f"{error_hint_position}\nHelp:\n{error_hint_help}"
|
||||
|
||||
if evo.project_path / Path(file_path) not in evo.files:
|
||||
continue
|
||||
error = CIError(
|
||||
raw_str=raw_str,
|
||||
file_path=file_path,
|
||||
line=int(line_number),
|
||||
column=int(column_number),
|
||||
code=error_code,
|
||||
msg=error_message,
|
||||
hint=error_hint,
|
||||
checker="mypy",
|
||||
)
|
||||
|
||||
errors[file_path].append(error)
|
||||
|
||||
return CIFeedback(errors=errors)
|
||||
|
||||
|
||||
class MultiEvaluator(Evaluator):
|
||||
def __init__(self, *evaluators: Evaluator) -> None:
|
||||
self.evaluators = evaluators
|
||||
|
||||
def evaluate(self, evo: Repo, **kwargs: dict) -> CIFeedback:
|
||||
all_errors = defaultdict(list)
|
||||
for evaluator in self.evaluators:
|
||||
feedback: CIFeedback = evaluator.evaluate(evo, **kwargs)
|
||||
for file_path, errors in feedback.errors.items():
|
||||
all_errors[file_path].extend(errors)
|
||||
|
||||
# sort errors by position
|
||||
for file_path in all_errors:
|
||||
all_errors[file_path].sort(key=lambda x: (x.line, x.column))
|
||||
|
||||
return CIFeedback(errors=all_errors)
|
||||
|
||||
|
||||
class CIEvoStr(EvolvingStrategy):
|
||||
def evolve( # noqa: C901, PLR0912, PLR0915
|
||||
self,
|
||||
evo: Repo,
|
||||
evolving_trace: list[EvoStep] | None = None,
|
||||
knowledge_l: list[Knowledge] | None = None,
|
||||
**kwargs: dict,
|
||||
) -> Repo:
|
||||
@dataclass
|
||||
class CodeFixGroup:
|
||||
start_line: int
|
||||
end_line: int
|
||||
errors: list[CIError]
|
||||
session_id: str
|
||||
responses: list[str]
|
||||
|
||||
api = APIBackend()
|
||||
system_prompt = CI_prompts["linting_system_prompt_template"].format(language="Python")
|
||||
|
||||
if len(evolving_trace) > 0:
|
||||
last_feedback: CIFeedback = evolving_trace[-1].feedback
|
||||
|
||||
# print statistics
|
||||
checker_error_counts = {
|
||||
checker: sum(c_statistics.values()) for checker, c_statistics in last_feedback.statistics().items()
|
||||
}
|
||||
print(
|
||||
f"Found [red]{sum(checker_error_counts.values())}[/red] errors, "
|
||||
"including: "
|
||||
+ ", ".join(
|
||||
f"[red]{count}[/red] [magenta]{checker}[/magenta] errors"
|
||||
for checker, count in checker_error_counts.items()
|
||||
),
|
||||
)
|
||||
|
||||
fix_records: dict[str, FixRecord] = defaultdict(
|
||||
lambda: FixRecord([], [], [], defaultdict(list)),
|
||||
)
|
||||
|
||||
# Group errors by code blocks
|
||||
fix_groups: dict[str, list[CodeFixGroup]] = defaultdict(list)
|
||||
changes: dict[str, list[tuple[int, int, str]]] = defaultdict(list)
|
||||
for file_path, errors in last_feedback.errors.items():
|
||||
file = evo.files[evo.project_path / Path(file_path)]
|
||||
|
||||
# check if the file needs to add `from __future__ import annotations`
|
||||
# need to add rules here for different languages/tools
|
||||
# TODO @bowen: current way of handling errors like 'Add import statement' may be not good
|
||||
for error in errors:
|
||||
if error.code in ("FA100", "FA102"):
|
||||
changes[file_path].append((1, 1, "from __future__ import annotations\n"))
|
||||
break
|
||||
|
||||
# Group errors by code blocks
|
||||
error_p = 0
|
||||
for start_line, end_line in file.get_code_blocks(max_lines=30):
|
||||
group_errors: list[CIError] = []
|
||||
|
||||
# collect errors in the same code block
|
||||
while error_p < len(errors) and start_line <= errors[error_p].line <= end_line:
|
||||
if errors[error_p].code not in ("FA100", "FA102"):
|
||||
group_errors.append(errors[error_p])
|
||||
error_p += 1
|
||||
|
||||
# process errors in the code block
|
||||
if group_errors:
|
||||
session = api.build_chat_session(session_system_prompt=system_prompt)
|
||||
session_id = session.get_conversation_id()
|
||||
session.build_chat_completion(
|
||||
CI_prompts["session_start_template"].format(code=file.get(add_line_number=True)),
|
||||
)
|
||||
|
||||
fix_groups[file_path].append(
|
||||
CodeFixGroup(start_line, end_line, group_errors, session_id, []),
|
||||
)
|
||||
|
||||
# Fix errors in each code block
|
||||
with Progress(SpinnerColumn(), *Progress.get_default_columns(), TimeElapsedColumn()) as progress:
|
||||
group_counts = sum([len(groups) for groups in fix_groups.values()])
|
||||
task_id = progress.add_task("Fixing repo...", total=group_counts)
|
||||
|
||||
for file_path in fix_groups:
|
||||
file = evo.files[evo.project_path / Path(file_path)]
|
||||
for code_fix_g in fix_groups[file_path]:
|
||||
start_line = code_fix_g.start_line
|
||||
end_line = code_fix_g.end_line
|
||||
group_errors = code_fix_g.errors
|
||||
code_snippet_with_lineno = file.get(
|
||||
start_line,
|
||||
end_line,
|
||||
add_line_number=True,
|
||||
return_list=False,
|
||||
)
|
||||
errors_str = "\n\n".join(str(e) for e in group_errors)
|
||||
|
||||
# ask LLM to repair current code snippet
|
||||
user_prompt = CI_prompts["session_normal_template"].format(
|
||||
code=code_snippet_with_lineno,
|
||||
lint_info=errors_str,
|
||||
start_line=start_line,
|
||||
end_line=end_line,
|
||||
start_lineno=start_line,
|
||||
)
|
||||
|
||||
session = api.build_chat_session(conversation_id=code_fix_g.session_id)
|
||||
res = session.build_chat_completion(user_prompt)
|
||||
code_fix_g.responses.append(res)
|
||||
progress.update(
|
||||
task_id,
|
||||
description=f"[green]Fixing[/green] [cyan]{file_path}[/cyan]...",
|
||||
advance=1,
|
||||
)
|
||||
|
||||
# Manual inspection and repair
|
||||
for file_path in last_feedback.errors:
|
||||
print(
|
||||
Rule(
|
||||
f"[bright_blue]Checking[/bright_blue] [cyan]{file_path}[/cyan]",
|
||||
style="bright_blue",
|
||||
align="left",
|
||||
characters=".",
|
||||
),
|
||||
)
|
||||
|
||||
file = evo.files[evo.project_path / Path(file_path)]
|
||||
|
||||
# generate changes
|
||||
for group_id, code_fix_g in enumerate(fix_groups[file_path], start=1):
|
||||
start_line, end_line, group_errors = code_fix_g.start_line, code_fix_g.end_line, code_fix_g.errors
|
||||
session = api.build_chat_session(conversation_id=code_fix_g.session_id)
|
||||
|
||||
print(f"[yellow]Checking part {group_id}...[/yellow]")
|
||||
|
||||
front_context = file.get(start_line - 3, start_line - 1)
|
||||
rear_context = file.get(end_line + 1, end_line + 3)
|
||||
front_context_with_lineno = file.get(start_line - 3, start_line - 1, add_line_number=True)
|
||||
rear_context_with_lineno = file.get(end_line + 1, end_line + 3, add_line_number=True)
|
||||
|
||||
code_snippet_with_lineno = file.get(start_line, end_line, add_line_number=True, return_list=False)
|
||||
|
||||
# print errors
|
||||
printed_errors_str = "\n".join(
|
||||
[
|
||||
f"[{error.checker}] {error.line: >{file.lineno_width}}:{error.column: <4}"
|
||||
f" {error.code} {error.msg}"
|
||||
for error in group_errors
|
||||
],
|
||||
)
|
||||
print(
|
||||
Panel.fit(
|
||||
Syntax(printed_errors_str, lexer="python", background_color="default"),
|
||||
title=f"{len(group_errors)} Errors",
|
||||
),
|
||||
)
|
||||
|
||||
# print original code
|
||||
table = Table(show_header=False, box=None)
|
||||
table.add_column()
|
||||
table.add_row(Syntax(front_context_with_lineno, lexer="python", background_color="default"))
|
||||
table.add_row(Rule(style="dark_orange"))
|
||||
table.add_row(Syntax(code_snippet_with_lineno, lexer="python", background_color="default"))
|
||||
table.add_row(Rule(style="dark_orange"))
|
||||
table.add_row(Syntax(rear_context_with_lineno, lexer="python", background_color="default"))
|
||||
print(Panel.fit(table, title="Original Code"))
|
||||
|
||||
res = code_fix_g.responses[0]
|
||||
code_snippet_lines = file.get(start_line, end_line, add_line_number=False, return_list=True)
|
||||
|
||||
while True:
|
||||
try:
|
||||
new_code = re.search(r".*```[Pp]ython\n(.*?)\n```.*", res, re.DOTALL).group(1)
|
||||
except (re.error, AttributeError) as exc:
|
||||
print(f"[red]Error when extract codes[/red]:\n {res}\nException: {exc}")
|
||||
try:
|
||||
fixed_errors_info = re.search(r".*```[Jj]son\n(.*?)\n```.*", res, re.DOTALL).group(1)
|
||||
fixed_errors_info = json.loads(fixed_errors_info)
|
||||
except AttributeError:
|
||||
fixed_errors_info = None
|
||||
except (json.JSONDecodeError, re.error) as exc:
|
||||
fixed_errors_info = None
|
||||
print(f"[red]Error when extracting fixed_errors[/red]: {exc}")
|
||||
|
||||
new_code = CodeFile.remove_line_number(new_code)
|
||||
|
||||
# print repair status (code diff)
|
||||
diff = ndiff(code_snippet_lines, new_code.split("\n"))
|
||||
|
||||
# add 2 spaces to align with diff format
|
||||
front_context = re.sub(r"^", " ", front_context, flags=re.MULTILINE)
|
||||
rear_context = re.sub(r"^", " ", rear_context, flags=re.MULTILINE)
|
||||
|
||||
table = Table(show_header=False, box=None)
|
||||
table.add_column()
|
||||
table.add_column()
|
||||
table.add_column()
|
||||
table.add_row("", "", Syntax(front_context, lexer="python", background_color="default"))
|
||||
table.add_row("", "", Rule(style="dark_orange"))
|
||||
diff_original_lineno = start_line
|
||||
diff_new_lineno = start_line
|
||||
for i in diff:
|
||||
if i.startswith("+"):
|
||||
table.add_row(
|
||||
"",
|
||||
Text(str(diff_new_lineno), style="green bold"),
|
||||
Text(i, style="green"),
|
||||
)
|
||||
diff_new_lineno += 1
|
||||
elif i.startswith("-"):
|
||||
table.add_row(
|
||||
Text(str(diff_original_lineno), style="red bold"),
|
||||
"",
|
||||
Text(i, style="red"),
|
||||
)
|
||||
diff_original_lineno += 1
|
||||
elif i.startswith("?"):
|
||||
table.add_row("", "", Text(i, style="yellow"))
|
||||
else:
|
||||
table.add_row(
|
||||
str(diff_original_lineno),
|
||||
str(diff_new_lineno),
|
||||
Syntax(i, lexer="python", background_color="default"),
|
||||
)
|
||||
diff_original_lineno += 1
|
||||
diff_new_lineno += 1
|
||||
table.add_row("", "", Rule(style="dark_orange"))
|
||||
table.add_row("", "", Syntax(rear_context, lexer="python", background_color="default"))
|
||||
print(Panel.fit(table, title="Repair Status"))
|
||||
|
||||
operation = Prompt.ask(
|
||||
"Input your operation [ [red]([bold]s[/bold])kip[/red] / "
|
||||
"[green]([bold]a[/bold])pply[/green] / "
|
||||
"[yellow]manual instruction[/yellow] ]",
|
||||
)
|
||||
print()
|
||||
if operation in ("s", "skip"):
|
||||
fix_records[file_path].skipped_errors.extend(group_errors)
|
||||
break
|
||||
if operation in ("a", "apply"):
|
||||
if fixed_errors_info:
|
||||
fixed_errors_str = "\n".join(fixed_errors_info["errors"])
|
||||
for error in group_errors:
|
||||
if f"{error.line}:{error.column}" in fixed_errors_str:
|
||||
fix_records[file_path].manually_fixed_errors.append(error)
|
||||
else:
|
||||
fix_records[file_path].skipped_errors.append(error)
|
||||
else:
|
||||
fix_records[file_path].directly_fixed_errors.extend(group_errors)
|
||||
|
||||
changes[file_path].append((start_line, end_line, new_code))
|
||||
break
|
||||
|
||||
fix_records[file_path].manual_instructions[operation].extend(group_errors)
|
||||
res = session.build_chat_completion(
|
||||
CI_prompts["session_manual_template"].format(operation=operation),
|
||||
)
|
||||
code_fix_g.responses.append(res)
|
||||
|
||||
# apply changes
|
||||
file.apply_changes(changes[file_path])
|
||||
|
||||
evo.fix_records = fix_records
|
||||
|
||||
return evo
|
||||
|
||||
|
||||
class CIEvoAgent(EvoAgent):
|
||||
def __init__(self, evolving_strategy: CIEvoStr) -> None:
|
||||
super().__init__(max_loop=1, evolving_strategy=evolving_strategy)
|
||||
self.evolving_trace = []
|
||||
|
||||
def multistep_evolve(self, evo: Repo, eva: Evaluator) -> Repo:
|
||||
evo = self.evolving_strategy.evolve(
|
||||
evo=evo,
|
||||
evolving_trace=self.evolving_trace,
|
||||
)
|
||||
|
||||
self.evolving_trace.append(EvoStep(evo, feedback=eva.evaluate(evo)))
|
||||
|
||||
return evo
|
||||
|
||||
|
||||
DIR = None
|
||||
while DIR is None or not DIR.exists():
|
||||
DIR = Prompt.ask("Please input the [cyan]project directory[/cyan]")
|
||||
DIR = Path(DIR)
|
||||
|
||||
excludes = Prompt.ask(
|
||||
"Input the [dark_orange]excluded directories[/dark_orange] (relative to "
|
||||
"[cyan]project path[/cyan] and separated by whitespace)",
|
||||
).split(" ")
|
||||
excludes = [Path(exclude.strip()) for exclude in excludes if exclude.strip() != ""]
|
||||
|
||||
start_time = time.time()
|
||||
start_timestamp = datetime.datetime.now(datetime.timezone.utc).strftime("%m%d%H%M")
|
||||
|
||||
repo = Repo(DIR, excludes=excludes)
|
||||
# evaluator = MultiEvaluator(MypyEvaluator(), RuffEvaluator())
|
||||
evaluator = RuffEvaluator()
|
||||
estr = CIEvoStr()
|
||||
ea = CIEvoAgent(estr)
|
||||
ea.multistep_evolve(repo, evaluator)
|
||||
while True:
|
||||
print(Rule(f"Round {len(ea.evolving_trace)} repair", style="blue"))
|
||||
repo: Repo = ea.multistep_evolve(repo, evaluator)
|
||||
|
||||
fix_records = repo.fix_records
|
||||
filename = f"{DIR.name}_{start_timestamp}_round_{len(ea.evolving_trace)}_fix_records.json"
|
||||
with Path(filename).open("w") as file:
|
||||
json.dump({k: v.to_dict() for k, v in fix_records.items()}, file, indent=4)
|
||||
|
||||
# Count the number of skipped errors
|
||||
skipped_errors_count = 0
|
||||
directly_fixed_errors_count = 0
|
||||
manually_fixed_errors_count = 0
|
||||
skipped_errors_code_count = defaultdict(int)
|
||||
directly_fixed_errors_code_count = defaultdict(int)
|
||||
manually_fixed_errors_code_count = defaultdict(int)
|
||||
code_message = defaultdict(str)
|
||||
for record in fix_records.values():
|
||||
skipped_errors_count += len(record.skipped_errors)
|
||||
directly_fixed_errors_count += len(record.directly_fixed_errors)
|
||||
manually_fixed_errors_count += len(record.manually_fixed_errors)
|
||||
for error in record.skipped_errors:
|
||||
skipped_errors_code_count[error.code] += 1
|
||||
code_message[error.code] = error.msg
|
||||
for error in record.directly_fixed_errors:
|
||||
directly_fixed_errors_code_count[error.code] += 1
|
||||
code_message[error.code] = error.msg
|
||||
for error in record.manually_fixed_errors:
|
||||
manually_fixed_errors_code_count[error.code] += 1
|
||||
code_message[error.code] = error.msg
|
||||
|
||||
skipped_errors_statistics = ""
|
||||
directly_fixed_errors_statistics = ""
|
||||
manually_fixed_errors_statistics = ""
|
||||
for code, count in sorted(skipped_errors_code_count.items(), key=lambda x: x[1], reverse=True):
|
||||
skipped_errors_statistics += f"{count: >5} {code: >10} {code_message[code]}\n"
|
||||
for code, count in sorted(directly_fixed_errors_code_count.items(), key=lambda x: x[1], reverse=True):
|
||||
directly_fixed_errors_statistics += f"{count: >5} {code: >10} {code_message[code]}\n"
|
||||
for code, count in sorted(manually_fixed_errors_code_count.items(), key=lambda x: x[1], reverse=True):
|
||||
manually_fixed_errors_statistics += f"{count: >5} {code: >10} {code_message[code]}\n"
|
||||
|
||||
# Create a table to display the counts and ratios
|
||||
table = Table(title="Error Fix Statistics")
|
||||
table.add_column("Type")
|
||||
table.add_column("Statistics")
|
||||
table.add_column("Count")
|
||||
table.add_column("Ratio")
|
||||
|
||||
total_errors_count = skipped_errors_count + directly_fixed_errors_count + manually_fixed_errors_count
|
||||
table.add_row("Total Errors", "", Text(str(total_errors_count), style="cyan"), "")
|
||||
table.add_row(
|
||||
Text("Skipped Errors", style="red"),
|
||||
skipped_errors_statistics,
|
||||
Text(str(skipped_errors_count), style="red"),
|
||||
Text(f"{skipped_errors_count / total_errors_count:.2%}"),
|
||||
style="red",
|
||||
)
|
||||
table.add_row(
|
||||
Text("Directly Fixed Errors", style="green"),
|
||||
directly_fixed_errors_statistics,
|
||||
Text(str(directly_fixed_errors_count), style="green"),
|
||||
Text(f"{directly_fixed_errors_count / total_errors_count:.2%}"),
|
||||
style="green",
|
||||
)
|
||||
table.add_row(
|
||||
Text("Manually Fixed Errors", style="yellow"),
|
||||
manually_fixed_errors_statistics,
|
||||
Text(str(manually_fixed_errors_count), style="yellow"),
|
||||
Text(f"{manually_fixed_errors_count / total_errors_count:.2%}"),
|
||||
style="yellow",
|
||||
)
|
||||
|
||||
print(table)
|
||||
operation = Prompt.ask("Start next round? (y/n)", choices=["y", "n"])
|
||||
if operation == "n":
|
||||
break
|
||||
|
||||
|
||||
end_time = time.time()
|
||||
execution_time = end_time - start_time
|
||||
print(f"Execution time: {execution_time} seconds")
|
||||
|
||||
""" Please commit it by hand... and then run the next round
|
||||
git add -u
|
||||
git commit --no-verify -v
|
||||
"""
|
||||
@@ -0,0 +1,225 @@
|
||||
import json
|
||||
import pickle
|
||||
from pathlib import Path
|
||||
|
||||
import fire
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import seaborn as sns
|
||||
|
||||
from rdagent.components.benchmark.conf import BenchmarkSettings
|
||||
from rdagent.components.benchmark.eval_method import FactorImplementEval
|
||||
|
||||
|
||||
class BenchmarkAnalyzer:
|
||||
def __init__(self, settings, only_correct_format=False):
|
||||
self.settings = settings
|
||||
self.index_map = self.load_index_map()
|
||||
self.only_correct_format = only_correct_format
|
||||
|
||||
def load_index_map(self):
|
||||
index_map = {}
|
||||
with open(self.settings.bench_data_path, "r") as file:
|
||||
factor_dict = json.load(file)
|
||||
for factor_name, data in factor_dict.items():
|
||||
index_map[factor_name] = (factor_name, data["Category"], data["Difficulty"])
|
||||
return index_map
|
||||
|
||||
def load_data(self, file_path):
|
||||
file_path = Path(file_path)
|
||||
if not (file_path.is_file() and file_path.suffix == ".pkl"):
|
||||
raise ValueError("Invalid file path")
|
||||
|
||||
with file_path.open("rb") as f:
|
||||
res = pickle.load(f)
|
||||
|
||||
return res
|
||||
|
||||
def process_results(self, results):
|
||||
final_res = {}
|
||||
for experiment, path in results.items():
|
||||
data = self.load_data(path)
|
||||
summarized_data = FactorImplementEval.summarize_res(data)
|
||||
processed_data = self.analyze_data(summarized_data)
|
||||
final_res[experiment] = processed_data.iloc[-1, :]
|
||||
return final_res
|
||||
|
||||
def reformat_index(self, display_df):
|
||||
"""
|
||||
reform the results from
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
success rate
|
||||
High_Beta_Factor 0.2
|
||||
|
||||
to
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
success rate
|
||||
Category Difficulty Factor
|
||||
量价 Hard High_Beta_Factor 0.2
|
||||
|
||||
"""
|
||||
new_idx = []
|
||||
display_df = display_df[display_df.index.isin(self.index_map.keys())]
|
||||
for idx in display_df.index:
|
||||
new_idx.append(self.index_map[idx])
|
||||
|
||||
display_df.index = pd.MultiIndex.from_tuples(
|
||||
new_idx,
|
||||
names=["Factor", "Category", "Difficulty"],
|
||||
)
|
||||
display_df = display_df.swaplevel(0, 2).swaplevel(0, 1).sort_index(axis=0)
|
||||
|
||||
return display_df.sort_index(
|
||||
key=lambda x: [{"Easy": 0, "Medium": 1, "Hard": 2, "New Discovery": 3}.get(i, i) for i in x]
|
||||
)
|
||||
|
||||
def result_all_key_order(self, x):
|
||||
order_v = []
|
||||
for i in x:
|
||||
order_v.append(
|
||||
{
|
||||
"Avg Run SR": 0,
|
||||
"Avg Format SR": 1,
|
||||
"Avg Correlation": 2,
|
||||
"Max Correlation": 3,
|
||||
"Max Accuracy": 4,
|
||||
"Avg Accuracy": 5,
|
||||
}.get(i, i),
|
||||
)
|
||||
return order_v
|
||||
|
||||
def analyze_data(self, sum_df):
|
||||
index = [
|
||||
"FactorSingleColumnEvaluator",
|
||||
"FactorRowCountEvaluator",
|
||||
"FactorIndexEvaluator",
|
||||
"FactorEqualValueRatioEvaluator",
|
||||
"FactorCorrelationEvaluator",
|
||||
"run factor error",
|
||||
]
|
||||
sum_df = sum_df.reindex(index, axis=0)
|
||||
sum_df_clean = sum_df.T.groupby(level=0).apply(lambda x: x.reset_index(drop=True))
|
||||
|
||||
run_error = sum_df_clean["run factor error"].unstack().T.fillna(False).astype(bool)
|
||||
succ_rate = ~run_error
|
||||
succ_rate = succ_rate.mean(axis=0).to_frame("success rate")
|
||||
|
||||
succ_rate_f = self.reformat_index(succ_rate)
|
||||
|
||||
# if it rasis Error when running the evaluator, we will get NaN
|
||||
# Running failures are reguarded to zero score.
|
||||
format_issue = sum_df_clean[["FactorRowCountEvaluator", "FactorIndexEvaluator"]].apply(
|
||||
lambda x: np.mean(x.fillna(0.0)), axis=1
|
||||
)
|
||||
format_succ_rate = format_issue.unstack().T.mean(axis=0).to_frame("success rate")
|
||||
format_succ_rate_f = self.reformat_index(format_succ_rate)
|
||||
|
||||
corr = sum_df_clean["FactorCorrelationEvaluator"].fillna(0.0)
|
||||
if self.only_correct_format:
|
||||
corr = corr.loc[format_issue == 1.0]
|
||||
|
||||
corr_res = corr.unstack().T.mean(axis=0).to_frame("corr(only success)")
|
||||
corr_res = self.reformat_index(corr_res)
|
||||
|
||||
corr_max = corr.unstack().T.max(axis=0).to_frame("corr(only success)")
|
||||
corr_max_res = self.reformat_index(corr_max)
|
||||
|
||||
value_max = sum_df_clean["FactorEqualValueRatioEvaluator"]
|
||||
value_max = value_max.unstack().T.max(axis=0).to_frame("max_value")
|
||||
value_max_res = self.reformat_index(value_max)
|
||||
|
||||
value_avg = (
|
||||
(sum_df_clean["FactorEqualValueRatioEvaluator"] * format_issue)
|
||||
.unstack()
|
||||
.T.mean(axis=0)
|
||||
.to_frame("avg_value")
|
||||
)
|
||||
value_avg_res = self.reformat_index(value_avg)
|
||||
|
||||
result_all = pd.concat(
|
||||
{
|
||||
"Avg Correlation": corr_res.iloc[:, 0],
|
||||
"Avg Format SR": format_succ_rate_f.iloc[:, 0],
|
||||
"Avg Run SR": succ_rate_f.iloc[:, 0],
|
||||
"Max Correlation": corr_max_res.iloc[:, 0],
|
||||
"Max Accuracy": value_max_res.iloc[:, 0],
|
||||
"Avg Accuracy": value_avg_res.iloc[:, 0],
|
||||
},
|
||||
axis=1,
|
||||
)
|
||||
|
||||
df = result_all.sort_index(axis=1, key=self.result_all_key_order).sort_index(axis=0)
|
||||
print(df)
|
||||
|
||||
print()
|
||||
print(df.groupby("Category").mean())
|
||||
|
||||
print()
|
||||
print(df.mean())
|
||||
|
||||
# Calculate the mean of each column
|
||||
mean_values = df.fillna(0.0).mean()
|
||||
mean_df = pd.DataFrame(mean_values).T
|
||||
|
||||
# Assign the MultiIndex to the DataFrame
|
||||
mean_df.index = pd.MultiIndex.from_tuples([("-", "-", "Average")], names=["Factor", "Category", "Difficulty"])
|
||||
|
||||
# Append the mean values to the end of the dataframe
|
||||
df_w_mean = pd.concat([df, mean_df]).astype("float")
|
||||
|
||||
return df_w_mean
|
||||
|
||||
|
||||
class Plotter:
|
||||
@staticmethod
|
||||
def change_fs(font_size):
|
||||
plt.rc("font", size=font_size)
|
||||
plt.rc("axes", titlesize=font_size)
|
||||
plt.rc("axes", labelsize=font_size)
|
||||
plt.rc("xtick", labelsize=font_size)
|
||||
plt.rc("ytick", labelsize=font_size)
|
||||
plt.rc("legend", fontsize=font_size)
|
||||
plt.rc("figure", titlesize=font_size)
|
||||
|
||||
@staticmethod
|
||||
def plot_data(data, file_name, title):
|
||||
plt.figure(figsize=(10, 10))
|
||||
plt.ylabel("Value")
|
||||
colors = ["#3274A1", "#E1812C", "#3A923A", "#C03D3E"]
|
||||
plt.bar(data["a"], data["b"], color=colors, capsize=5)
|
||||
for idx, row in data.iterrows():
|
||||
plt.text(idx, row["b"] + 0.01, f"{row['b']:.2f}", ha="center", va="bottom")
|
||||
plt.suptitle(title, y=0.98)
|
||||
plt.xticks(rotation=45)
|
||||
plt.ylim(0, 1)
|
||||
plt.tight_layout()
|
||||
plt.savefig(file_name)
|
||||
|
||||
|
||||
def main(
|
||||
path="git_ignore_folder/eval_results/res_promptV220240724-060037.pkl",
|
||||
round=1,
|
||||
title="Comparison of Different Methods",
|
||||
only_correct_format=False,
|
||||
):
|
||||
settings = BenchmarkSettings()
|
||||
benchmark = BenchmarkAnalyzer(settings, only_correct_format=only_correct_format)
|
||||
results = {
|
||||
f"{round} round experiment": path,
|
||||
}
|
||||
final_results = benchmark.process_results(results)
|
||||
final_results_df = pd.DataFrame(final_results)
|
||||
|
||||
Plotter.change_fs(20)
|
||||
plot_data = final_results_df.drop(["Max Accuracy", "Avg Accuracy"], axis=0).T
|
||||
plot_data = plot_data.reset_index().melt("index", var_name="a", value_name="b")
|
||||
Plotter.plot_data(plot_data, "./comparison_plot.png", title)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
fire.Fire(main)
|
||||
@@ -0,0 +1,35 @@
|
||||
from rdagent.app.qlib_rd_loop.conf import FACTOR_PROP_SETTING
|
||||
from rdagent.components.benchmark.conf import BenchmarkSettings
|
||||
from rdagent.components.benchmark.eval_method import FactorImplementEval
|
||||
from rdagent.core.scenario import Scenario
|
||||
from rdagent.core.utils import import_class
|
||||
from rdagent.log import rdagent_logger as logger
|
||||
from rdagent.scenarios.qlib.factor_experiment_loader.json_loader import (
|
||||
FactorTestCaseLoaderFromJsonFile,
|
||||
)
|
||||
|
||||
if __name__ == "__main__":
|
||||
# 1.read the settings
|
||||
bs = BenchmarkSettings()
|
||||
|
||||
# 2.read and prepare the eval_data
|
||||
test_cases = FactorTestCaseLoaderFromJsonFile().load(bs.bench_data_path)
|
||||
|
||||
# 3.declare the method to be tested and pass the arguments.
|
||||
|
||||
scen: Scenario = import_class(FACTOR_PROP_SETTING.scen)()
|
||||
generate_method = import_class(bs.bench_method_cls)(scen=scen, **bs.bench_method_extra_kwargs)
|
||||
# 4.declare the eval method and pass the arguments.
|
||||
eval_method = FactorImplementEval(
|
||||
method=generate_method,
|
||||
test_cases=test_cases,
|
||||
scen=scen,
|
||||
catch_eval_except=True,
|
||||
test_round=bs.bench_test_round,
|
||||
)
|
||||
|
||||
# 5.run the eval
|
||||
res = eval_method.eval(eval_method.develop())
|
||||
|
||||
# 6.save the result
|
||||
logger.log_object(res)
|
||||
@@ -0,0 +1,30 @@
|
||||
# Tasks
|
||||
|
||||
## Task Extraction
|
||||
From paper to task.
|
||||
```bash
|
||||
# python rdagent/app/model_implementation/task_extraction.py
|
||||
# It may based on rdagent/document_reader/document_reader.py
|
||||
python rdagent/components/task_implementation/model_implementation/task_extraction.py ./PaperImpBench/raw_paper/
|
||||
```
|
||||
|
||||
## Complete workflow
|
||||
From paper to implementation
|
||||
``` bash
|
||||
# Similar to
|
||||
# rdagent/app/factor_extraction_and_implementation/factor_extract_and_implement.py
|
||||
```
|
||||
|
||||
## Paper benchmark
|
||||
```bash
|
||||
# TODO: it does not work well now.
|
||||
python rdagent/app/model_implementation/eval.py
|
||||
```
|
||||
|
||||
TODO:
|
||||
- Create reasonable benchmark
|
||||
- with uniform input
|
||||
- manually create task
|
||||
- Create reasonable evaluation metrics
|
||||
|
||||
## Evolving
|
||||
@@ -0,0 +1,42 @@
|
||||
from pathlib import Path
|
||||
|
||||
from rdagent.components.coder.model_coder import ModelCoSTEER
|
||||
from rdagent.components.loader.task_loader import ModelTaskLoaderJson, ModelWsLoader
|
||||
from rdagent.scenarios.qlib.experiment.model_experiment import (
|
||||
QlibModelExperiment,
|
||||
QlibModelScenario,
|
||||
)
|
||||
|
||||
if __name__ == "__main__":
|
||||
DIRNAME = Path(__file__).absolute().resolve().parent
|
||||
|
||||
from rdagent.components.coder.model_coder.benchmark.eval import ModelImpValEval
|
||||
from rdagent.components.coder.model_coder.one_shot import ModelCodeWriter
|
||||
|
||||
bench_folder = DIRNAME.parent.parent.parent / "components" / "coder" / "model_coder" / "benchmark"
|
||||
mtl = ModelTaskLoaderJson(str(bench_folder / "model_dict.json"))
|
||||
|
||||
task_l = mtl.load()
|
||||
|
||||
task_l = [t for t in task_l if t.name == "A-DGN"] # FIXME: other models does not work well
|
||||
|
||||
model_experiment = QlibModelExperiment(sub_tasks=task_l)
|
||||
# mtg = ModelCodeWriter(scen=QlibModelScenario())
|
||||
mtg = ModelCoSTEER(scen=QlibModelScenario())
|
||||
|
||||
model_experiment = mtg.develop(model_experiment)
|
||||
|
||||
# TODO: Align it with the benchmark framework after @wenjun's refine the evaluation part.
|
||||
# Currently, we just handcraft a workflow for fast evaluation.
|
||||
|
||||
mil = ModelWsLoader(bench_folder / "gt_code")
|
||||
|
||||
mie = ModelImpValEval()
|
||||
# Evaluation:
|
||||
eval_l = []
|
||||
for impl in model_experiment.sub_workspace_list:
|
||||
print(impl.target_task)
|
||||
gt_impl = mil.load(impl.target_task)
|
||||
eval_l.append(mie.evaluate(gt_impl, impl))
|
||||
|
||||
print(eval_l)
|
||||
@@ -0,0 +1,203 @@
|
||||
"""
|
||||
CLI entrance for all rdagent application.
|
||||
|
||||
This will
|
||||
- make rdagent a nice entry and
|
||||
- autoamtically load dotenv
|
||||
"""
|
||||
|
||||
import sys
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv(".env")
|
||||
# 1) Make sure it is at the beginning of the script so that it will load dotenv before initializing BaseSettings.
|
||||
# 2) The ".env" argument is necessary to make sure it loads `.env` from the current directory.
|
||||
|
||||
import subprocess
|
||||
from importlib.resources import path as rpath
|
||||
from typing import Optional
|
||||
|
||||
import typer
|
||||
from typing_extensions import Annotated
|
||||
|
||||
from rdagent.app.data_science.loop import main as data_science
|
||||
from rdagent.app.finetune.llm.loop import main as llm_finetune
|
||||
from rdagent.app.general_model.general_model import (
|
||||
extract_models_and_implement as general_model,
|
||||
)
|
||||
from rdagent.app.qlib_rd_loop.factor import main as fin_factor
|
||||
from rdagent.app.qlib_rd_loop.factor_from_report import main as fin_factor_report
|
||||
from rdagent.app.qlib_rd_loop.model import main as fin_model
|
||||
from rdagent.app.qlib_rd_loop.quant import main as fin_quant
|
||||
from rdagent.app.utils.health_check import health_check
|
||||
from rdagent.app.utils.info import collect_info
|
||||
from rdagent.log.mle_summary import grade_summary as grade_summary
|
||||
|
||||
app = typer.Typer()
|
||||
|
||||
CheckoutOption = Annotated[bool, typer.Option("--checkout/--no-checkout", "-c/-C")]
|
||||
CheckEnvOption = Annotated[bool, typer.Option("--check-env/--no-check-env", "-e/-E")]
|
||||
CheckDockerOption = Annotated[bool, typer.Option("--check-docker/--no-check-docker", "-d/-D")]
|
||||
CheckPortsOption = Annotated[bool, typer.Option("--check-ports/--no-check-ports", "-p/-P")]
|
||||
|
||||
|
||||
def ui(port=19899, log_dir="", debug: bool = False, data_science: bool = False):
|
||||
"""
|
||||
start web app to show the log traces.
|
||||
"""
|
||||
if data_science:
|
||||
with rpath("rdagent.log.ui", "dsapp.py") as app_path:
|
||||
cmds = ["streamlit", "run", app_path, f"--server.port={port}"]
|
||||
subprocess.run(cmds)
|
||||
return
|
||||
with rpath("rdagent.log.ui", "app.py") as app_path:
|
||||
cmds = ["streamlit", "run", app_path, f"--server.port={port}"]
|
||||
if log_dir or debug:
|
||||
cmds.append("--")
|
||||
if log_dir:
|
||||
cmds.append(f"--log_dir={log_dir}")
|
||||
if debug:
|
||||
cmds.append("--debug")
|
||||
subprocess.run(cmds)
|
||||
|
||||
|
||||
def server_ui(port=19899):
|
||||
"""
|
||||
start the Flask log server in real time
|
||||
"""
|
||||
from rdagent.log.server.app import main as log_server_main
|
||||
|
||||
log_server_main(port=port)
|
||||
|
||||
|
||||
def ds_user_interact(port=19900):
|
||||
"""
|
||||
start web app to show the log traces in real time
|
||||
"""
|
||||
commands = ["streamlit", "run", "rdagent/log/ui/ds_user_interact.py", f"--server.port={port}"]
|
||||
subprocess.run(commands)
|
||||
|
||||
|
||||
@app.command(name="fin_factor")
|
||||
def fin_factor_cli(
|
||||
path: Optional[str] = None,
|
||||
step_n: Optional[int] = None,
|
||||
loop_n: Optional[int] = None,
|
||||
all_duration: Optional[str] = None,
|
||||
checkout: CheckoutOption = True,
|
||||
):
|
||||
fin_factor(path=path, step_n=step_n, loop_n=loop_n, all_duration=all_duration, checkout=checkout)
|
||||
|
||||
|
||||
@app.command(name="fin_model")
|
||||
def fin_model_cli(
|
||||
path: Optional[str] = None,
|
||||
step_n: Optional[int] = None,
|
||||
loop_n: Optional[int] = None,
|
||||
all_duration: Optional[str] = None,
|
||||
checkout: CheckoutOption = True,
|
||||
):
|
||||
fin_model(path=path, step_n=step_n, loop_n=loop_n, all_duration=all_duration, checkout=checkout)
|
||||
|
||||
|
||||
@app.command(name="fin_quant")
|
||||
def fin_quant_cli(
|
||||
path: Optional[str] = None,
|
||||
step_n: Optional[int] = None,
|
||||
loop_n: Optional[int] = None,
|
||||
all_duration: Optional[str] = None,
|
||||
checkout: CheckoutOption = True,
|
||||
):
|
||||
fin_quant(path=path, step_n=step_n, loop_n=loop_n, all_duration=all_duration, checkout=checkout)
|
||||
|
||||
|
||||
@app.command(name="fin_factor_report")
|
||||
def fin_factor_report_cli(
|
||||
report_folder: Optional[str] = None,
|
||||
path: Optional[str] = None,
|
||||
all_duration: Optional[str] = None,
|
||||
checkout: CheckoutOption = True,
|
||||
):
|
||||
fin_factor_report(report_folder=report_folder, path=path, all_duration=all_duration, checkout=checkout)
|
||||
|
||||
|
||||
@app.command(name="general_model")
|
||||
def general_model_cli(report_file_path: str):
|
||||
general_model(report_file_path)
|
||||
|
||||
|
||||
@app.command(name="data_science")
|
||||
def data_science_cli(
|
||||
path: Optional[str] = None,
|
||||
checkout: CheckoutOption = True,
|
||||
step_n: Optional[int] = None,
|
||||
loop_n: Optional[int] = None,
|
||||
timeout: Optional[str] = None,
|
||||
competition: Optional[str] = None,
|
||||
):
|
||||
data_science(
|
||||
path=path,
|
||||
checkout=checkout,
|
||||
step_n=step_n,
|
||||
loop_n=loop_n,
|
||||
timeout=timeout,
|
||||
competition=competition,
|
||||
)
|
||||
|
||||
|
||||
@app.command(name="llm_finetune")
|
||||
def llm_finetune_cli(
|
||||
path: Optional[str] = None,
|
||||
checkout: CheckoutOption = True,
|
||||
benchmark: Optional[str] = None,
|
||||
benchmark_description: Optional[str] = None,
|
||||
dataset: Optional[str] = None,
|
||||
base_model: Optional[str] = None,
|
||||
upper_data_size_limit: Optional[int] = None,
|
||||
step_n: Optional[int] = None,
|
||||
loop_n: Optional[int] = None,
|
||||
timeout: Optional[str] = None,
|
||||
):
|
||||
llm_finetune(
|
||||
path=path,
|
||||
checkout=checkout,
|
||||
benchmark=benchmark,
|
||||
benchmark_description=benchmark_description,
|
||||
dataset=dataset,
|
||||
base_model=base_model,
|
||||
upper_data_size_limit=upper_data_size_limit,
|
||||
step_n=step_n,
|
||||
loop_n=loop_n,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
|
||||
@app.command(name="grade_summary")
|
||||
def grade_summary_cli(log_folder: str):
|
||||
grade_summary(log_folder)
|
||||
|
||||
|
||||
app.command(name="ui")(ui)
|
||||
app.command(name="server_ui")(server_ui)
|
||||
|
||||
|
||||
@app.command(name="health_check")
|
||||
def health_check_cli(
|
||||
check_env: CheckEnvOption = True,
|
||||
check_docker: CheckDockerOption = True,
|
||||
check_ports: CheckPortsOption = True,
|
||||
):
|
||||
health_check(check_env=check_env, check_docker=check_docker, check_ports=check_ports)
|
||||
|
||||
|
||||
@app.command(name="collect_info")
|
||||
def collect_info_cli():
|
||||
collect_info()
|
||||
|
||||
|
||||
app.command(name="ds_user_interact")(ds_user_interact)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
app()
|
||||
@@ -0,0 +1,206 @@
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
from pydantic_settings import SettingsConfigDict
|
||||
|
||||
from rdagent.app.kaggle.conf import KaggleBasePropSetting
|
||||
|
||||
|
||||
class DataScienceBasePropSetting(KaggleBasePropSetting):
|
||||
# TODO: Kaggle Setting should be the subclass of DataScience
|
||||
model_config = SettingsConfigDict(env_prefix="DS_", protected_namespaces=())
|
||||
|
||||
# Main components
|
||||
## Scen
|
||||
scen: str = "rdagent.scenarios.data_science.scen.KaggleScen"
|
||||
"""
|
||||
Scenario class for data science tasks.
|
||||
- For Kaggle competitions, use: "rdagent.scenarios.data_science.scen.KaggleScen"
|
||||
- For custom data science scenarios, use: "rdagent.scenarios.data_science.scen.DataScienceScen"
|
||||
"""
|
||||
|
||||
planner: str = "rdagent.scenarios.data_science.proposal.exp_gen.planner.DSExpPlannerHandCraft"
|
||||
hypothesis_gen: str = "rdagent.scenarios.data_science.proposal.exp_gen.router.ParallelMultiTraceExpGen"
|
||||
interactor: str = "rdagent.components.interactor.SkipInteractor"
|
||||
trace_scheduler: str = "rdagent.scenarios.data_science.proposal.exp_gen.trace_scheduler.RoundRobinScheduler"
|
||||
"""Hypothesis generation class"""
|
||||
|
||||
summarizer: str = "rdagent.scenarios.data_science.dev.feedback.DSExperiment2Feedback"
|
||||
summarizer_init_kwargs: dict = {
|
||||
"version": "exp_feedback",
|
||||
}
|
||||
## Workflow Related
|
||||
consecutive_errors: int = 5
|
||||
|
||||
## Coding Related
|
||||
coding_fail_reanalyze_threshold: int = 3
|
||||
|
||||
debug_recommend_timeout: int = 600
|
||||
"""The recommend time limit for running on debugging data"""
|
||||
debug_timeout: int = 600
|
||||
"""The timeout limit for running on debugging data"""
|
||||
full_recommend_timeout: int = 3600
|
||||
"""The recommend time limit for running on full data"""
|
||||
full_timeout: int = 3600
|
||||
"""The timeout limit for running on full data"""
|
||||
|
||||
#### model dump
|
||||
enable_model_dump: bool = False
|
||||
enable_doc_dev: bool = False
|
||||
model_dump_check_level: Literal["medium", "high"] = "medium"
|
||||
|
||||
#### MCP documentation search integration
|
||||
enable_mcp_documentation_search: bool = False
|
||||
"""Enable MCP documentation search for error resolution. Requires MCP_ENABLED=true and MCP_CONTEXT7_ENABLED=true in environment."""
|
||||
|
||||
### specific feature
|
||||
|
||||
### notebook integration
|
||||
enable_notebook_conversion: bool = False
|
||||
|
||||
#### enable specification
|
||||
spec_enabled: bool = True
|
||||
|
||||
#### proposal related
|
||||
# proposal_version: str = "v2" deprecated
|
||||
|
||||
coder_on_whole_pipeline: bool = True
|
||||
max_trace_hist: int = 3
|
||||
|
||||
coder_max_loop: int = 10
|
||||
runner_max_loop: int = 3
|
||||
|
||||
sample_data_by_LLM: bool = True
|
||||
use_raw_description: bool = False
|
||||
show_nan_columns: bool = False
|
||||
|
||||
### knowledge base
|
||||
enable_knowledge_base: bool = False
|
||||
knowledge_base_version: str = "v1"
|
||||
knowledge_base_path: str | None = None
|
||||
idea_pool_json_path: str | None = None
|
||||
|
||||
### archive log folder after each loop
|
||||
enable_log_archive: bool = True
|
||||
log_archive_path: str | None = None
|
||||
log_archive_temp_path: str | None = (
|
||||
None # This is to store the mid tar file since writing the tar file is preferred in local storage then copy to target storage
|
||||
)
|
||||
|
||||
#### Evaluation on Test related
|
||||
eval_sub_dir: str = "eval" # TODO: fixme, this is not a good name
|
||||
"""We'll use f"{DS_RD_SETTING.local_data_path}/{DS_RD_SETTING.eval_sub_dir}/{competition}"
|
||||
to find the scriipt to evaluate the submission on test"""
|
||||
|
||||
"""---below are the settings for multi-trace---"""
|
||||
|
||||
### multi-trace related
|
||||
max_trace_num: int = 1
|
||||
"""The maximum number of traces to grow before merging"""
|
||||
|
||||
scheduler_temperature: float = 1.0
|
||||
"""The temperature for the trace scheduler for softmax calculation, used in ProbabilisticScheduler"""
|
||||
|
||||
# PUCT exploration constant for MCTSScheduler (ignored by other schedulers)
|
||||
scheduler_c_puct: float = 1.0
|
||||
"""Exploration constant used by MCTSScheduler (PUCT)."""
|
||||
|
||||
enable_score_reward: bool = False
|
||||
"""Enable using score-based reward for trace selection in multi-trace scheduling."""
|
||||
|
||||
#### multi-trace:checkpoint selector
|
||||
selector_name: str = "rdagent.scenarios.data_science.proposal.exp_gen.select.expand.LatestCKPSelector"
|
||||
"""The name of the selector to use"""
|
||||
sota_count_window: int = 5
|
||||
"""The number of trials to consider for SOTA count"""
|
||||
sota_count_threshold: int = 1
|
||||
"""The threshold for SOTA count"""
|
||||
|
||||
#### multi-trace: SOTA experiment selector
|
||||
sota_exp_selector_name: str = "rdagent.scenarios.data_science.proposal.exp_gen.select.submit.GlobalSOTASelector"
|
||||
"""The name of the SOTA experiment selector to use"""
|
||||
|
||||
### multi-trace:inject optimals for multi-trace
|
||||
# inject diverse when start a new sub-trace
|
||||
enable_inject_diverse: bool = False
|
||||
|
||||
# inject diverse from other traces when start a new sub-trace
|
||||
enable_cross_trace_diversity: bool = True
|
||||
"""Enable cross-trace diversity injection when starting a new sub-trace.
|
||||
This is different from `enable_inject_diverse` which is for non-parallel cases."""
|
||||
|
||||
diversity_injection_strategy: str = (
|
||||
"rdagent.scenarios.data_science.proposal.exp_gen.diversity_strategy.InjectUntilSOTAGainedStrategy"
|
||||
)
|
||||
"""The strategy to use for injecting diversity context."""
|
||||
|
||||
# enable different version of DSExpGen for multi-trace
|
||||
enable_multi_version_exp_gen: bool = False
|
||||
exp_gen_version_list: str = "v3,v2"
|
||||
|
||||
#### multi-trace: time for final multi-trace merge
|
||||
merge_hours: float = 0
|
||||
"""The time for merge"""
|
||||
|
||||
#### multi-trace: max SOTA-retrieved number, used in AutoSOTAexpSelector
|
||||
# constrains the number of SOTA experiments to retrieve, otherwise too many SOTA experiments to retrieve will cause the exceed of the context window of LLM
|
||||
max_sota_retrieved_num: int = 10
|
||||
"""The maximum number of SOTA experiments to retrieve in a LLM call"""
|
||||
|
||||
#### enable draft before first sota experiment
|
||||
enable_draft_before_first_sota: bool = False
|
||||
enable_planner: bool = False
|
||||
|
||||
model_architecture_suggestion_time_percent: float = 0.75
|
||||
allow_longer_timeout: bool = False
|
||||
coder_enable_llm_decide_longer_timeout: bool = False
|
||||
runner_enable_llm_decide_longer_timeout: bool = False
|
||||
coder_longer_timeout_multiplier_upper: int = 3
|
||||
runner_longer_timeout_multiplier_upper: int = 2
|
||||
coder_timeout_increase_stage: float = 0.3
|
||||
runner_timeout_increase_stage: float = 0.3
|
||||
runner_timeout_increase_stage_patience: int = 2
|
||||
"""Number of failures tolerated before escalating to next timeout level (stage width). Every 'patience' failures, timeout increases by 'runner_timeout_increase_stage'"""
|
||||
show_hard_limit: bool = True
|
||||
|
||||
#### enable runner code change summary
|
||||
runner_enable_code_change_summary: bool = True
|
||||
|
||||
### Proposal workflow related
|
||||
|
||||
#### Hypothesis Generate related
|
||||
enable_simple_hypothesis: bool = False
|
||||
"""If true, generate simple hypothesis, no more than 2 sentences each."""
|
||||
|
||||
enable_generate_unique_hypothesis: bool = False
|
||||
"""Enable generate unique hypothesis. If True, generate unique hypothesis for each component. If False, generate unique hypothesis for each component."""
|
||||
|
||||
enable_research_rag: bool = False
|
||||
"""Enable research RAG for hypothesis generation."""
|
||||
|
||||
#### hypothesis critique and rewrite
|
||||
enable_hypo_critique_rewrite: bool = False
|
||||
"""Enable hypothesis critique and rewrite stages for improving hypothesis quality"""
|
||||
enable_scale_check: bool = False
|
||||
|
||||
##### select related
|
||||
ratio_merge_or_ensemble: int = 70
|
||||
"""The ratio of merge or ensemble to be considered as a valid solution"""
|
||||
llm_select_hypothesis: bool = False
|
||||
"""Whether to use LLM to select hypothesis. If True, use LLM selection; if False, use the existing ranking method."""
|
||||
|
||||
#### Task Generate related
|
||||
fix_seed_and_data_split: bool = False
|
||||
|
||||
ensemble_time_upper_bound: bool = False
|
||||
|
||||
user_interaction_wait_seconds: int = 6000 # seconds to wait for user interaction
|
||||
user_interaction_mid_folder: Path = Path.cwd() / "git_ignore_folder" / "RD-Agent_user_interaction"
|
||||
|
||||
|
||||
DS_RD_SETTING = DataScienceBasePropSetting()
|
||||
|
||||
# enable_cross_trace_diversity and llm_select_hypothesis should not be true at the same time
|
||||
assert not (
|
||||
DS_RD_SETTING.enable_cross_trace_diversity and DS_RD_SETTING.llm_select_hypothesis
|
||||
), "enable_cross_trace_diversity and llm_select_hypothesis cannot be true at the same time"
|
||||
@@ -0,0 +1,6 @@
|
||||
import fire
|
||||
|
||||
from rdagent.scenarios.data_science.debug.data import create_debug_data
|
||||
|
||||
if __name__ == "__main__":
|
||||
fire.Fire(create_debug_data)
|
||||
@@ -0,0 +1,83 @@
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import fire
|
||||
|
||||
from rdagent.app.data_science.conf import DS_RD_SETTING
|
||||
from rdagent.core.utils import import_class
|
||||
from rdagent.log import rdagent_logger as logger
|
||||
from rdagent.scenarios.data_science.loop import DataScienceRDLoop
|
||||
|
||||
|
||||
def main(
|
||||
path: Optional[str] = None,
|
||||
checkout: bool = True,
|
||||
checkout_path: Optional[str] = None,
|
||||
step_n: Optional[int] = None,
|
||||
loop_n: Optional[int] = None,
|
||||
timeout: Optional[str] = None,
|
||||
competition="bms-molecular-translation",
|
||||
replace_timer=True,
|
||||
exp_gen_cls: Optional[str] = None,
|
||||
):
|
||||
"""
|
||||
|
||||
Parameters
|
||||
----------
|
||||
path :
|
||||
A path like `$LOG_PATH/__session__/1/0_propose`. This indicates that we restore the state after finishing step 0 in loop 1.
|
||||
checkout :
|
||||
Used to control the log session path. Boolean type, default is True.
|
||||
- If True, the new loop will use the existing folder and clear logs for sessions after the one corresponding to the given path.
|
||||
- If False, the new loop will use the existing folder but keep the logs for sessions after the one corresponding to the given path.
|
||||
checkout_path:
|
||||
If a checkout_path (or a str like Path) is provided, the new loop will be saved to that path, leaving the original path unchanged.
|
||||
step_n :
|
||||
Number of steps to run; if None, the process will run indefinitely until an error or KeyboardInterrupt occurs.
|
||||
loop_n :
|
||||
Number of loops to run; if None, the process will run indefinitely until an error or KeyboardInterrupt occurs.
|
||||
- If the current loop is incomplete, it will be counted as the first loop for completion.
|
||||
- If both step_n and loop_n are provided, the process will stop as soon as either condition is met.
|
||||
timeout :
|
||||
Maximum duration to run the loop. Accepts a string format recognized by the internal timer.
|
||||
- If None, the loop will run until completion, error, or KeyboardInterrupt.
|
||||
competition :
|
||||
Competition name.
|
||||
replace_timer :
|
||||
If a session is loaded, determines whether to replace the timer with session.timer.
|
||||
exp_gen_cls :
|
||||
When there are different stages, the exp_gen can be replaced with the new proposal.
|
||||
|
||||
|
||||
Auto R&D Evolving loop for models in a Kaggle scenario.
|
||||
You can continue running a session by using the command:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
dotenv run -- python rdagent/app/data_science/loop.py [--competition titanic] $LOG_PATH/__session__/1/0_propose --step_n 1 # `step_n` is an optional parameter
|
||||
rdagent kaggle --competition playground-series-s4e8 # This command is recommended.
|
||||
"""
|
||||
if not checkout_path is None:
|
||||
checkout = Path(checkout_path)
|
||||
|
||||
if competition is not None:
|
||||
DS_RD_SETTING.competition = competition
|
||||
|
||||
if not DS_RD_SETTING.competition:
|
||||
logger.error("Please specify competition name.")
|
||||
|
||||
if path is None:
|
||||
kaggle_loop = DataScienceRDLoop(DS_RD_SETTING)
|
||||
else:
|
||||
kaggle_loop: DataScienceRDLoop = DataScienceRDLoop.load(path, checkout=checkout, replace_timer=replace_timer)
|
||||
|
||||
# replace exp_gen if we have new class
|
||||
if exp_gen_cls is not None:
|
||||
kaggle_loop.exp_gen = import_class(exp_gen_cls)(kaggle_loop.exp_gen.scen)
|
||||
|
||||
asyncio.run(kaggle_loop.run(step_n=step_n, loop_n=loop_n, all_duration=timeout))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
fire.Fire(main)
|
||||
@@ -0,0 +1,40 @@
|
||||
import os
|
||||
|
||||
from pydantic_settings import SettingsConfigDict
|
||||
|
||||
from rdagent.app.data_science.conf import DS_RD_SETTING
|
||||
from rdagent.core.conf import RD_AGENT_SETTINGS, ExtendedBaseSettings
|
||||
|
||||
|
||||
class DSFinetuneScen(ExtendedBaseSettings):
|
||||
model_config = SettingsConfigDict(env_prefix="FT_", protected_namespaces=())
|
||||
scen: str = "rdagent.app.finetune.data_science.scen.DSFinetuneScen"
|
||||
"""
|
||||
Scenario class for data science tasks.
|
||||
- For Kaggle competitions, use: "rdagent.scenarios.data_science.scen.KaggleScen"
|
||||
- For custom data science scenarios, use: "rdagent.scenarios.data_science.scen.DataScienceScen"
|
||||
- For LLM finetune scenarios, use: "rdagent.app.finetune.llm.scen.LLMFinetuneScen"
|
||||
- For Data science finetune scenarios, use: "rdagent.app.finetune.data_science.scen.DSFinetuneScen"
|
||||
"""
|
||||
|
||||
debug_timeout: int = 3600
|
||||
"""The timeout limit for running on debugging data"""
|
||||
full_timeout: int = 10800
|
||||
"""The timeout limit for running on full data"""
|
||||
|
||||
coder_on_whole_pipeline: bool = True
|
||||
enable_model_dump: bool = True
|
||||
app_tpl: str = "app/finetune/data_science/tpl"
|
||||
|
||||
|
||||
def update_settings(competition: str):
|
||||
"""
|
||||
Update the RD_AGENT_SETTINGS with the values from DS_FINETUNE_SETTINGS.
|
||||
"""
|
||||
DS_FINETUNE_SETTINGS = DSFinetuneScen()
|
||||
RD_AGENT_SETTINGS.app_tpl = DS_FINETUNE_SETTINGS.app_tpl
|
||||
os.environ["DS_CODER_COSTEER_EXTRA_EVALUATOR"] = '["rdagent.app.finetune.share.eval.PrevModelLoadEvaluator"]'
|
||||
for field_name, new_value in DS_FINETUNE_SETTINGS.model_dump().items():
|
||||
if hasattr(DS_RD_SETTING, field_name):
|
||||
setattr(DS_RD_SETTING, field_name, new_value)
|
||||
DS_RD_SETTING.competition = competition
|
||||
@@ -0,0 +1,40 @@
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
|
||||
import fire
|
||||
|
||||
from rdagent.app.data_science.conf import DS_RD_SETTING
|
||||
from rdagent.app.finetune.data_science.conf import update_settings
|
||||
from rdagent.core.utils import import_class
|
||||
from rdagent.log import rdagent_logger as logger
|
||||
from rdagent.scenarios.data_science.loop import DataScienceRDLoop
|
||||
|
||||
|
||||
def main(
|
||||
model: str | None = None,
|
||||
competition: str | None = None,
|
||||
):
|
||||
"""
|
||||
Parameters
|
||||
----------
|
||||
competition :
|
||||
Competition name.
|
||||
|
||||
Auto R&D Evolving loop for models finetune.
|
||||
You can continue running a session by using the command:
|
||||
.. code-block:: bash
|
||||
dotenv run -- python rdagent/app/finetune/data_science/loop.py --competition aerial-cactus-identification
|
||||
"""
|
||||
if not competition:
|
||||
raise Exception("Please specify competition name.")
|
||||
|
||||
model_folder = Path(DS_RD_SETTING.local_data_path) / competition / "prev_model"
|
||||
if not model_folder.exists():
|
||||
raise Exception(f"Please put the model path to {model_folder}.")
|
||||
update_settings(competition)
|
||||
rd_loop: DataScienceRDLoop = DataScienceRDLoop(DS_RD_SETTING)
|
||||
asyncio.run(rd_loop.run())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
fire.Fire(main)
|
||||
@@ -0,0 +1,20 @@
|
||||
from pathlib import Path
|
||||
|
||||
from rdagent.app.data_science.conf import DS_RD_SETTING
|
||||
from rdagent.core.scenario import Scenario
|
||||
from rdagent.log import rdagent_logger as logger
|
||||
from rdagent.scenarios.data_science.scen import DataScienceScen
|
||||
from rdagent.scenarios.data_science.scen.utils import describe_data_folder_v2
|
||||
from rdagent.utils.agent.tpl import T
|
||||
|
||||
|
||||
class DSFinetuneScen(DataScienceScen):
|
||||
"""DSFinetuneScen Scenario"""
|
||||
|
||||
def _get_data_folder_description(self) -> str:
|
||||
folder_desc = describe_data_folder_v2(
|
||||
Path(DS_RD_SETTING.local_data_path) / self.competition,
|
||||
show_nan_columns=DS_RD_SETTING.show_nan_columns,
|
||||
max_length=20000, # more context for model script
|
||||
)
|
||||
return folder_desc
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
pipeline_coder:
|
||||
system: |-
|
||||
{% include "rdagent.components.coder.data_science.pipeline.prompts:pipeline_coder.system" %}
|
||||
NOTE: Ensure that base model form `{% include "scenarios.data_science.share:scen.input_path" %}prev_model` is correctly loaded, you are supposed to finetune the base model.
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
task_gen:
|
||||
system: |-
|
||||
{% include "rdagent.scenarios.data_science.proposal.exp_gen.prompts_v2:task_gen.system" %}
|
||||
NOTE: You MUST load base model form `{% include "scenarios.data_science.share:scen.input_path" %}prev_model`. Your main goal is to finetune it.
|
||||
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
# FT-Agent: Autonomous LLM Fine-Tuning
|
||||
|
||||
This directory contains the RD-Agent LLM fine-tuning scenario used by **FT-Agent** in the ICML 2026 paper [FT-Dojo: Towards Autonomous LLM Fine-Tuning with Language Agents](https://arxiv.org/abs/2603.01712).
|
||||
|
||||
FT-Agent automates a benchmark-driven fine-tuning loop:
|
||||
|
||||
1. inspect the target benchmark and available raw datasets;
|
||||
2. generate data processing code and LLaMA-Factory training configs;
|
||||
3. run fail-fast validation before full training;
|
||||
4. fine-tune the target model;
|
||||
5. evaluate with OpenCompass and use feedback for the next iteration.
|
||||
|
||||
The implementation is research-oriented. A full run can download large datasets, build training/evaluation environments, call LLM APIs for data processing, and consume GPU hours.
|
||||
|
||||
## Supported Benchmarks
|
||||
|
||||
The scenario currently includes benchmark adapters for the FT-Dojo tasks and several related extensions.
|
||||
|
||||
| Domain | Benchmarks | Main raw dataset |
|
||||
| --- | --- | --- |
|
||||
| Math | `aime24`, `aime25` | `deepscaler` |
|
||||
| Patent | `panorama_par4pc`, `panorama_pi4pc`, `panorama_noc4pc` | `panorama` |
|
||||
| Chemistry | `chemcotbench_mol_und`, `chemcotbench_mol_edit`, `chemcotbench_mol_opt`, `chemcotbench_reaction` | `chemcot` |
|
||||
| Table QA | `tablebench_data_analysis`, `tablebench_fact_checking`, `tablebench_numerical_reasoning`, `tablebench_visualization` | `tableinstruct` |
|
||||
| Finance | `FinanceIQ_gen` | `financeiq` |
|
||||
| Biology | `bioprobench_gen`, `bioprobench_ord`, `bioprobench_err`, `bioprobench_pqa` | `bioprobench` |
|
||||
|
||||
Dataset registration lives in `rdagent/scenarios/finetune/datasets/__init__.py`. Benchmark adapters live in `rdagent/scenarios/finetune/benchmark/data/adaptor.py`.
|
||||
|
||||
Current dataset behavior: scenario startup prepares the registered dataset resources under `FT_FILE_PATH`. The first run can therefore be large and slow. Reusing the same `FT_FILE_PATH` lets later runs reuse already downloaded assets. The `--dataset` option constrains the agent's selected dataset after preparation; it does not change the current preparation step.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Linux.
|
||||
- Docker available to the current user without `sudo`, or a compatible conda setup.
|
||||
- NVIDIA GPU for realistic fine-tuning runs.
|
||||
- LLM API access for the RD-Agent planner and optional data processing calls.
|
||||
- Hugging Face access for target models and datasets. Some datasets may require accepting upstream licenses or setting `HF_TOKEN`.
|
||||
|
||||
Install RD-Agent from source when using this scenario:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/microsoft/RD-Agent
|
||||
cd RD-Agent
|
||||
make dev
|
||||
```
|
||||
|
||||
## Minimal `.env`
|
||||
|
||||
Create `.env` in the repository root. Adjust model names and API settings to your provider.
|
||||
|
||||
```bash
|
||||
# LLM backend
|
||||
BACKEND=rdagent.oai.backend.LiteLLMAPIBackend
|
||||
CHAT_MODEL=gpt-4o
|
||||
CHAT_TEMPERATURE=1
|
||||
CHAT_STREAM=True
|
||||
OPENAI_API_KEY=<your_api_key>
|
||||
# OPENAI_API_BASE=<your_api_base_if_needed>
|
||||
|
||||
# Embedding model used by RD-Agent infrastructure
|
||||
EMBEDDING_MODEL=text-embedding-3-small
|
||||
|
||||
# Fine-tuning workspace. Keep this stable to reuse downloaded models/datasets.
|
||||
FT_FILE_PATH=/absolute/path/to/finetune_files
|
||||
|
||||
# Runtime environment: docker is the default path; conda is available for local setups.
|
||||
FT_Coder_CoSTEER_env_type=docker
|
||||
|
||||
# Target task. You may also pass these through CLI arguments.
|
||||
FT_TARGET_BENCHMARK=aime25
|
||||
FT_BENCHMARK_DESCRIPTION="AIME 2025 math competition problems. Each answer is an integer from 0 to 999. Expected Output Format: put the final answer within \\boxed{}, for example \\boxed{42}."
|
||||
|
||||
# Target model and data-processing settings
|
||||
FT_BASE_MODEL=Qwen/Qwen2.5-7B-Instruct
|
||||
FT_UPPER_DATA_SIZE_LIMIT=2000
|
||||
FT_API_MAX_WORKERS=8
|
||||
FT_STRONG_MODELS='["gpt-4o"]'
|
||||
FT_WEAK_MODELS='["gpt-4o-mini"]'
|
||||
|
||||
# Hugging Face token, if required by a model or dataset.
|
||||
# HF_TOKEN=<your_hf_token>
|
||||
```
|
||||
|
||||
`FT_API_MAX_WORKERS` defaults to `8` to avoid surprising rate-limit and cost spikes for public users. If you have a high-throughput internal endpoint, increase it explicitly in `.env`.
|
||||
|
||||
## Single-Task Run
|
||||
|
||||
Using the CLI entry point:
|
||||
|
||||
```bash
|
||||
rdagent llm_finetune \
|
||||
--benchmark aime25 \
|
||||
--benchmark-description "AIME 2025 math competition problems. Each answer is an integer from 0 to 999. Expected Output Format: put the final answer within \\boxed{}, for example \\boxed{42}." \
|
||||
--base-model Qwen/Qwen2.5-7B-Instruct \
|
||||
--loop-n 3 \
|
||||
--timeout 12h
|
||||
```
|
||||
|
||||
Equivalent direct Python entry point:
|
||||
|
||||
```bash
|
||||
dotenv run -- python rdagent/app/finetune/llm/loop.py \
|
||||
--benchmark aime25 \
|
||||
--benchmark-description "AIME 2025 math competition problems. Each answer is an integer from 0 to 999. Expected Output Format: put the final answer within \\boxed{}, for example \\boxed{42}." \
|
||||
--base-model Qwen/Qwen2.5-7B-Instruct \
|
||||
--loop-n 3 \
|
||||
--timeout 12h
|
||||
```
|
||||
|
||||
Useful arguments:
|
||||
|
||||
| Argument | Meaning |
|
||||
| --- | --- |
|
||||
| `--base-model` | Hugging Face model id to fine-tune. Required unless set by `FT_BASE_MODEL`. |
|
||||
| `--benchmark` | Target benchmark key, such as `aime25` or `chemcotbench_mol_edit`. |
|
||||
| `--benchmark-description` | Natural-language task and output-format description. Required unless set in `.env`. |
|
||||
| `--dataset` | Dataset name to select for the agent after dataset preparation. |
|
||||
| `--upper-data-size-limit` | Maximum number of training examples used by one experiment. |
|
||||
| `--loop-n` | Maximum number of RD-Agent loops. |
|
||||
| `--timeout` | Overall wall-clock budget, such as `12h`. |
|
||||
|
||||
## Batch Runs
|
||||
|
||||
For multiple benchmark/model runs, use the job helper in this directory:
|
||||
|
||||
```bash
|
||||
cp rdagent/app/finetune/llm/job/tasks.json.example rdagent/app/finetune/llm/job/tasks.json
|
||||
cp .env rdagent/app/finetune/llm/job/.env
|
||||
bash rdagent/app/finetune/llm/job/run_ft_job.sh rdagent/app/finetune/llm/job/tasks.json
|
||||
```
|
||||
|
||||
The job runner reads benchmark descriptions from `rdagent/app/finetune/llm/job/scenarios.json` when a task does not provide `benchmark_description` directly.
|
||||
|
||||
## Logs and UI
|
||||
|
||||
Runs write RD-Agent traces under the configured log directory. For the Streamlit FT UI:
|
||||
|
||||
```bash
|
||||
streamlit run rdagent/app/finetune/llm/ui/app.py
|
||||
```
|
||||
|
||||
For the generic RD-Agent log UI:
|
||||
|
||||
```bash
|
||||
rdagent ui --port 19899 --log-dir <your_log_folder>
|
||||
```
|
||||
|
||||
## Notes
|
||||
|
||||
- The first run is expected to be slow because it may download models, datasets, LLaMA-Factory assets, and OpenCompass assets.
|
||||
- Docker mode mounts models and datasets from `FT_FILE_PATH` into the training/evaluation environments.
|
||||
- Generated training data must use Alpaca-style `instruction`, `input`, and `output` fields; the validator checks this before full training.
|
||||
- Evaluation uses validation feedback for agent iteration and keeps the test split for final reporting/front-end display.
|
||||
@@ -0,0 +1,125 @@
|
||||
from pathlib import Path
|
||||
|
||||
from pydantic_settings import SettingsConfigDict
|
||||
|
||||
from rdagent.core.conf import ExtendedBaseSettings
|
||||
|
||||
|
||||
class LLMFinetunePropSetting(ExtendedBaseSettings):
|
||||
"""LLM Fine-tune dedicated property settings.
|
||||
|
||||
- Adjust timeouts and template
|
||||
- Use FT_ env prefix for overrides
|
||||
"""
|
||||
|
||||
model_config = SettingsConfigDict(env_prefix="FT_", protected_namespaces=())
|
||||
|
||||
# Main Components
|
||||
scen: str = "rdagent.scenarios.finetune.scen.scenario.LLMFinetuneScen"
|
||||
"""Scenario class for LLM fine-tuning tasks."""
|
||||
|
||||
hypothesis_gen: str = "rdagent.scenarios.finetune.proposal.proposal.LLMFinetuneExpGen"
|
||||
"""Hypothesis generation class for LLM fine-tuning tasks."""
|
||||
|
||||
coder: str = "rdagent.components.coder.finetune.LLMFinetuneCoSTEER"
|
||||
"""Code generator.
|
||||
Function: Generate LLM fine-tuning code based on experiment design.
|
||||
"""
|
||||
|
||||
runner: str = "rdagent.scenarios.finetune.train.runner.LLMFinetuneRunner" # TODO
|
||||
"""Code runner.
|
||||
Function: Execute LLM fine-tuning code in a Docker environment.
|
||||
"""
|
||||
|
||||
summarizer: str = "rdagent.scenarios.finetune.dev.feedback.FTExperiment2Feedback"
|
||||
"""Result summarizer - To be implemented.
|
||||
Function: Analyze fine-tuning results and generate feedback, including performance metrics and error analysis.
|
||||
"""
|
||||
|
||||
# Timeouts (longer for LLM training, all for Docker container timeout)
|
||||
full_timeout: int = 360000
|
||||
"""Full training timeout in seconds (default 100 hours, env: FT_FULL_TIMEOUT). Used in running stage for complete model training."""
|
||||
data_processing_timeout: int = 3600
|
||||
"""Data processing script timeout in seconds (default 1 hour, env: FT_DATA_PROCESSING_TIMEOUT). Used for full data processing in running stage."""
|
||||
debug_data_processing_timeout: int = 1200
|
||||
"""Debug data processing timeout in seconds (default 20 minutes, env: FT_DEBUG_DATA_PROCESSING_TIMEOUT). Used for --debug mode in coding stage."""
|
||||
micro_batch_timeout: int = 1800
|
||||
"""Micro-batch test timeout in seconds (default 30 minutes, env: FT_MICRO_BATCH_TIMEOUT)."""
|
||||
|
||||
# Pipeline behavior
|
||||
coder_on_whole_pipeline: bool = True
|
||||
app_tpl: str = "scenarios/finetune"
|
||||
|
||||
# Benchmark evaluation (always enabled as part of evaluation pipeline)
|
||||
|
||||
benchmark_timeout: int = 0
|
||||
"""Benchmark evaluation timeout in seconds. 0 means no timeout."""
|
||||
|
||||
# Judge API configuration (for llmjudge benchmarks like AIME)
|
||||
judge_model: str = "gpt-5.1"
|
||||
"""LLM judge model name for evaluation"""
|
||||
|
||||
judge_api_key: str | None = None
|
||||
"""API key for judge model (if None, will try to use from environment)"""
|
||||
|
||||
judge_api_base: str | None = None
|
||||
"""API base URL for judge model (if None, will use default)"""
|
||||
|
||||
judge_retry: int = 10
|
||||
"""Number of retries for LLM judge API calls (env: FT_JUDGE_RETRY)"""
|
||||
|
||||
benchmark_limit: int | None = None
|
||||
"""Limit number of samples for benchmark evaluation (None for full evaluation). Use for quick testing and debugging."""
|
||||
|
||||
benchmark_num_runs: int = 1
|
||||
"""Number of times to run each sample (for computing average or pass@k). Set >1 for multiple runs."""
|
||||
|
||||
benchmark_pass_k: list[int] | None = None
|
||||
"""Pass@k parameter list for code generation tasks (e.g., [1, 5, 10]). None to disable."""
|
||||
|
||||
# Data paths and processing
|
||||
file_path: Path = Path.cwd() / "git_ignore_folder" / "finetune_files"
|
||||
show_nan_columns: bool = False
|
||||
sample_data_by_LLM: bool = True
|
||||
|
||||
# LLM-specific fields
|
||||
user_target_scenario: str | None = None
|
||||
target_benchmark: str | None = None
|
||||
"""Benchmark dataset to evaluate on. Supported: aime25, aime24, math, mmlu, etc."""
|
||||
benchmark_description: str | None = None
|
||||
base_model: str | None = None
|
||||
dataset: str | None = None
|
||||
upper_data_size_limit: int = 2000
|
||||
|
||||
# Data processing LLM models (for API calls in data processing scripts)
|
||||
strong_models: list[str] = ["gpt-5", "gpt-5.1"]
|
||||
"""Strong models for complex tasks (CoT generation, reasoning) - supports list (env: FT_STRONG_MODELS)"""
|
||||
|
||||
weak_models: list[str] = ["gpt-4o-mini", "o4-mini", "gpt-5-mini"]
|
||||
"""Weak models for simple tasks (filtering, format conversion) - supports list (env: FT_WEAK_MODELS)"""
|
||||
|
||||
embedding_models: list[str] = ["text-embedding-3-small", "text-embedding-3-large"]
|
||||
|
||||
# Docker settings
|
||||
docker_enable_cache: bool = False
|
||||
"""Enable Docker cache for training (set via FT_DOCKER_ENABLE_CACHE)"""
|
||||
|
||||
# data sample count
|
||||
data_sample_count: int = 3
|
||||
|
||||
# API concurrency for data processing
|
||||
api_max_workers: int = 8
|
||||
"""Max concurrent workers for LLM API calls in data processing scripts (env: FT_API_MAX_WORKERS)"""
|
||||
|
||||
# Coder settings
|
||||
coder_max_loop: int = 10
|
||||
|
||||
# CoT format settings
|
||||
force_think_token: bool = False
|
||||
"""Force <think> token wrapping for CoT training data (env: FT_FORCE_THINK_TOKEN).
|
||||
When True: Data must be wrapped in <think>...</think> format, benchmark uses extract-non-reasoning-content postprocessor.
|
||||
When False: CoT reasoning required but format is flexible, no postprocessor needed."""
|
||||
|
||||
|
||||
# Global setting instance for LLM finetuning scenario
|
||||
FT_RD_SETTING = LLMFinetunePropSetting()
|
||||
@@ -0,0 +1,80 @@
|
||||
# FT Job Runner
|
||||
|
||||
`run_ft_job.sh` launches multiple FT-Agent runs in parallel under one job directory. It is a convenience helper for multi-GPU machines; for a single run, prefer the command in `../README.md`.
|
||||
|
||||
## Quick Start
|
||||
|
||||
From the RD-Agent repository root:
|
||||
|
||||
```bash
|
||||
# 1. Prepare the normal FT-Agent config first.
|
||||
# See rdagent/app/finetune/llm/README.md for required values.
|
||||
cp .env rdagent/app/finetune/llm/job/.env
|
||||
|
||||
# 2. Prepare task definitions.
|
||||
cp rdagent/app/finetune/llm/job/tasks.json.example rdagent/app/finetune/llm/job/tasks.json
|
||||
|
||||
# 3. Run the job.
|
||||
bash rdagent/app/finetune/llm/job/run_ft_job.sh rdagent/app/finetune/llm/job/tasks.json
|
||||
```
|
||||
|
||||
The script opens one tmux window per task and writes logs under `log/<job_id>/`.
|
||||
|
||||
## Requirements
|
||||
|
||||
- `jq`
|
||||
- `tmux`
|
||||
- `conda`
|
||||
- an RD-Agent conda environment, named `rdagent` by default
|
||||
|
||||
If your RD-Agent environment has a different name, set `CONDA_ENV_NAME` before running the script or add it to `job/.env`:
|
||||
|
||||
```bash
|
||||
CONDA_ENV_NAME=my-rdagent-env bash rdagent/app/finetune/llm/job/run_ft_job.sh
|
||||
```
|
||||
|
||||
The FT training/evaluation backend is still controlled by `FT_Coder_CoSTEER_env_type` in `job/.env`. In `docker` mode, the job runner skips conda readiness checks for the training and OpenCompass environments. In `conda` mode, it waits for the `llm_finetune` and `opencompass` environments after the first task initializes them.
|
||||
|
||||
## Task Config
|
||||
|
||||
`tasks.json` contains a list of independent runs:
|
||||
|
||||
```json
|
||||
{
|
||||
"tasks": [
|
||||
{
|
||||
"model": "Qwen/Qwen2.5-7B-Instruct",
|
||||
"benchmark": "aime25",
|
||||
"gpus": "0,1",
|
||||
"timeout": "12h"
|
||||
},
|
||||
{
|
||||
"model": "Qwen/Qwen2.5-7B-Instruct",
|
||||
"benchmark": "chemcotbench_mol_edit",
|
||||
"gpus": "2,3",
|
||||
"benchmark_description": "Molecule Editing - perform valid SMILES edits and return JSON: {\"output\": \"<valid SMILES>\"}."
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
| Field | Required | Default | Description |
|
||||
| --- | --- | --- | --- |
|
||||
| `model` | Yes | - | Hugging Face model id. |
|
||||
| `benchmark` | Yes | - | Benchmark key, such as `aime25` or `chemcotbench_mol_edit`. |
|
||||
| `gpus` | No | `"0"` | Value for `CUDA_VISIBLE_DEVICES`. |
|
||||
| `timeout` | No | `"12h"` | Wall-clock budget passed to the FT-Agent loop. |
|
||||
| `port` | No | - | Optional local API port; writes `OPENAI_API_BASE=http://localhost:<port>` for that task. |
|
||||
| `benchmark_description` | No | from `scenarios.json` | Task and output-format description. |
|
||||
|
||||
If `benchmark_description` is omitted, the runner looks it up in `scenarios.json`.
|
||||
|
||||
## Monitoring
|
||||
|
||||
```bash
|
||||
tmux attach -t rdagent
|
||||
tail -f log/<job_id>/*.log
|
||||
streamlit run rdagent/app/finetune/llm/ui/app.py
|
||||
```
|
||||
|
||||
In the Streamlit UI, select the generated job folder as the log source.
|
||||
Executable
+229
@@ -0,0 +1,229 @@
|
||||
#!/bin/bash
|
||||
# Run multiple FT tasks in parallel under a single job directory
|
||||
#
|
||||
# Usage: ./run_ft_job.sh [tasks.json]
|
||||
#
|
||||
# Config format (tasks.json):
|
||||
# {
|
||||
# "tasks": [
|
||||
# {"model": "Qwen/Qwen2.5-7B-Instruct", "benchmark": "aime25", "gpus": "0,1"},
|
||||
# {"model": "Qwen/Qwen2.5-7B-Instruct", "benchmark": "chemcotbench_mol_edit", "gpus": "2,3"}
|
||||
# ]
|
||||
# }
|
||||
|
||||
set -e
|
||||
|
||||
# ========== CONFIG ==========
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
RDAGENT_DIR="$(cd "$SCRIPT_DIR/../../../../.." && pwd)"
|
||||
ENV_FILE="$SCRIPT_DIR/.env"
|
||||
SCENARIOS_FILE="$SCRIPT_DIR/scenarios.json"
|
||||
STAGGER_DELAY=60
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 [tasks.json]"
|
||||
echo "Run multiple FT tasks under a single job directory."
|
||||
echo "UI: streamlit run rdagent/app/finetune/llm/ui/app.py"
|
||||
exit 0
|
||||
}
|
||||
|
||||
# ========== PARSE ARGS ==========
|
||||
CONFIG_FILE=""
|
||||
|
||||
for arg in "$@"; do
|
||||
case $arg in
|
||||
-h|--help) usage ;;
|
||||
*) [[ -z "$CONFIG_FILE" ]] && CONFIG_FILE="$arg" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
[[ -z "$CONFIG_FILE" ]] && CONFIG_FILE="$SCRIPT_DIR/tasks.json"
|
||||
[[ ! -f "$CONFIG_FILE" ]] && echo "Error: Config not found: $CONFIG_FILE" && exit 1
|
||||
|
||||
# Check .env file
|
||||
if [[ ! -f "$ENV_FILE" ]]; then
|
||||
echo "Error: .env not found at $ENV_FILE"
|
||||
echo "Create it from your repository root config, for example:"
|
||||
echo " cp $RDAGENT_DIR/.env $ENV_FILE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check jq
|
||||
command -v jq &>/dev/null || { echo "Error: jq required"; exit 1; }
|
||||
command -v tmux &>/dev/null || { echo "Error: tmux required"; exit 1; }
|
||||
|
||||
# Read the FT runtime mode from the job env file for readiness checks.
|
||||
FT_ENV_TYPE=$(grep -E "^FT_Coder_CoSTEER_env_type=" "$ENV_FILE" | tail -1 | cut -d= -f2- | tr -d "\"'" || true)
|
||||
FT_ENV_TYPE="${FT_ENV_TYPE:-docker}"
|
||||
|
||||
# ========== SETUP ==========
|
||||
# Get log and workspace base paths from environment or use defaults
|
||||
# Default to project-relative paths; can be overridden by environment variables
|
||||
FT_LOG_BASE="${FT_LOG_BASE:-$RDAGENT_DIR/log}"
|
||||
FT_WORKSPACE_BASE="${FT_WORKSPACE_BASE:-$RDAGENT_DIR/git_ignore_folder/RD-Agent_workspace}"
|
||||
|
||||
JOB_ID=$(date +%Y-%m-%d_%H-%M)
|
||||
JOB_DIR="$FT_LOG_BASE/$JOB_ID"
|
||||
if [[ -d "$JOB_DIR" ]]; then
|
||||
i=1; while [[ -d "${JOB_DIR}_$i" ]]; do ((i++)); done
|
||||
JOB_ID="${JOB_ID}_$i"; JOB_DIR="${JOB_DIR}_$i"
|
||||
fi
|
||||
mkdir -p "$JOB_DIR"
|
||||
|
||||
cd "$RDAGENT_DIR"
|
||||
|
||||
NUM_TASKS=$(jq '.tasks | length' "$CONFIG_FILE")
|
||||
|
||||
echo "=============================================="
|
||||
echo "FT Job: $JOB_ID"
|
||||
echo "=============================================="
|
||||
echo "Config: $CONFIG_FILE"
|
||||
echo "Tasks: $NUM_TASKS"
|
||||
echo "Log: $JOB_DIR"
|
||||
echo "Workspace: $FT_WORKSPACE_BASE/$JOB_ID"
|
||||
echo "FT env: $FT_ENV_TYPE"
|
||||
echo ""
|
||||
|
||||
# Setup tmux session
|
||||
TMUX_SESSION="rdagent"
|
||||
tmux kill-session -t "$TMUX_SESSION" 2>/dev/null || true
|
||||
tmux new-session -d -s "$TMUX_SESSION" -n "main"
|
||||
echo "Tmux session created: $TMUX_SESSION"
|
||||
echo ""
|
||||
|
||||
for ((i=0; i<NUM_TASKS; i++)); do
|
||||
model=$(jq -r ".tasks[$i].model" "$CONFIG_FILE")
|
||||
benchmark=$(jq -r ".tasks[$i].benchmark" "$CONFIG_FILE")
|
||||
gpus=$(jq -r ".tasks[$i].gpus // \"0\"" "$CONFIG_FILE")
|
||||
port=$(jq -r ".tasks[$i].port // empty" "$CONFIG_FILE")
|
||||
task_timeout=$(jq -r ".tasks[$i].timeout // \"12h\"" "$CONFIG_FILE")
|
||||
|
||||
# Load benchmark_description: tasks.json -> scenarios.json
|
||||
benchmark_desc=$(jq -r ".tasks[$i].benchmark_description // empty" "$CONFIG_FILE")
|
||||
if [[ -z "$benchmark_desc" ]]; then
|
||||
benchmark_desc=$(jq -r ".[\"$benchmark\"].benchmark_description // empty" "$SCENARIOS_FILE")
|
||||
fi
|
||||
if [[ -z "$benchmark_desc" ]]; then
|
||||
echo "Error: missing benchmark_description for benchmark '$benchmark'."
|
||||
echo "Add it to the task entry in $CONFIG_FILE or register it in $SCENARIOS_FILE."
|
||||
exit 1
|
||||
fi
|
||||
# Note: Special characters in benchmark_desc are handled by writing to env file
|
||||
model_name=$(basename "$model")
|
||||
task_name="${benchmark}_${model_name}"
|
||||
trace_path="$JOB_DIR/$task_name"
|
||||
|
||||
port_info=""
|
||||
[[ -n "$port" ]] && port_info=", port=$port"
|
||||
echo "Task $i: $task_name (model=$model, benchmark=$benchmark, gpus=$gpus$port_info)"
|
||||
|
||||
# Run task in tmux window with script -c for output capture
|
||||
task_workspace="$FT_WORKSPACE_BASE/$JOB_ID/$task_name"
|
||||
mkdir -p "$task_workspace"
|
||||
LOG_FILE="$JOB_DIR/${task_name}.log"
|
||||
|
||||
# Write task-specific env file (avoids command-line escaping issues with special chars)
|
||||
TASK_ENV_FILE="$task_workspace/.task_env"
|
||||
cat > "$TASK_ENV_FILE" << EOF
|
||||
CUDA_VISIBLE_DEVICES='$gpus'
|
||||
LOG_TRACE_PATH='$trace_path'
|
||||
WORKSPACE_PATH='$task_workspace'
|
||||
FT_TARGET_BENCHMARK='$benchmark'
|
||||
EOF
|
||||
if [[ -n "${CONDA_ENV_NAME:-}" ]]; then
|
||||
printf "CONDA_ENV_NAME=%q\n" "$CONDA_ENV_NAME" >> "$TASK_ENV_FILE"
|
||||
fi
|
||||
# Escape shell special characters for double-quoted string: \ " ` $
|
||||
if [[ -n "$benchmark_desc" ]]; then
|
||||
escaped_desc="$benchmark_desc"
|
||||
escaped_desc="${escaped_desc//\\/\\\\}" # \ -> \\
|
||||
escaped_desc="${escaped_desc//\"/\\\"}" # " -> \"
|
||||
escaped_desc="${escaped_desc//\`/\\\`}" # ` -> \`
|
||||
escaped_desc="${escaped_desc//\$/\\\$}" # $ -> \$
|
||||
echo "FT_BENCHMARK_DESCRIPTION=\"$escaped_desc\"" >> "$TASK_ENV_FILE"
|
||||
fi
|
||||
[[ -n "$port" ]] && echo "OPENAI_API_BASE='http://localhost:$port'" >> "$TASK_ENV_FILE"
|
||||
|
||||
# Create tmux window for this task and get its full target (e.g., rdagent:1.0)
|
||||
# Use "session:" format to ensure window is created in the correct session
|
||||
WIN_TARGET=$(tmux new-window -t "$TMUX_SESSION:" -n "$benchmark" -P)
|
||||
|
||||
# Build the command with environment setup (env vars loaded from file)
|
||||
timeout_arg=""
|
||||
[[ -n "$task_timeout" ]] && timeout_arg="--timeout $task_timeout"
|
||||
|
||||
TASK_RUN_SCRIPT="$task_workspace/run_task.sh"
|
||||
cat > "$TASK_RUN_SCRIPT" << EOF
|
||||
#!/bin/bash
|
||||
set -e
|
||||
set -a
|
||||
source "$ENV_FILE"
|
||||
source "$TASK_ENV_FILE"
|
||||
set +a
|
||||
|
||||
CONDA_ENV_NAME="\${CONDA_ENV_NAME:-rdagent}"
|
||||
if command -v conda >/dev/null 2>&1; then
|
||||
eval "\$(conda shell.bash hook)"
|
||||
elif [ -f "\$HOME/miniconda3/etc/profile.d/conda.sh" ]; then
|
||||
source "\$HOME/miniconda3/etc/profile.d/conda.sh"
|
||||
elif [ -f "\$HOME/miniforge3/etc/profile.d/conda.sh" ]; then
|
||||
source "\$HOME/miniforge3/etc/profile.d/conda.sh"
|
||||
elif [ -f "\$HOME/anaconda3/etc/profile.d/conda.sh" ]; then
|
||||
source "\$HOME/anaconda3/etc/profile.d/conda.sh"
|
||||
else
|
||||
echo "Error: conda not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
conda activate "\$CONDA_ENV_NAME"
|
||||
cd "$RDAGENT_DIR"
|
||||
python rdagent/app/finetune/llm/loop.py --base-model "$model" $timeout_arg
|
||||
EOF
|
||||
chmod +x "$TASK_RUN_SCRIPT"
|
||||
|
||||
# Run with script -c to capture terminal output (using full target for reliability)
|
||||
tmux send-keys -t "$WIN_TARGET" "script -q '$LOG_FILE' -c 'bash \"$TASK_RUN_SCRIPT\"'" Enter
|
||||
|
||||
echo " Window: $benchmark"
|
||||
echo ""
|
||||
|
||||
# Stagger starts
|
||||
if [[ $i -eq 0 ]]; then
|
||||
# First task: wait for initialization
|
||||
# Get FT_FILE_PATH from .env or use default
|
||||
FT_FILE_PATH=$(grep -E "^FT_FILE_PATH=" "$ENV_FILE" | cut -d= -f2 | tr -d '"' || echo "")
|
||||
[[ -z "$FT_FILE_PATH" ]] && FT_FILE_PATH="$RDAGENT_DIR/git_ignore_folder/finetune"
|
||||
DATASET_INFO="$FT_FILE_PATH/datasets/dataset_info.json"
|
||||
|
||||
echo " Waiting for scenario initialization (dataset_info.json)..."
|
||||
while [[ ! -f "$DATASET_INFO" ]]; do
|
||||
sleep 5
|
||||
done
|
||||
echo " Scenario initialized!"
|
||||
|
||||
if [[ "$FT_ENV_TYPE" == "conda" ]]; then
|
||||
echo " Waiting for llm_finetune conda env..."
|
||||
while ! conda run -n llm_finetune python -c "import requests" 2>/dev/null; do
|
||||
sleep 10
|
||||
done
|
||||
|
||||
echo " Waiting for opencompass conda env..."
|
||||
while ! conda run -n opencompass python -c "import opencompass" 2>/dev/null; do
|
||||
sleep 10
|
||||
done
|
||||
echo " Conda environments ready!"
|
||||
else
|
||||
echo " Docker mode detected; skipping conda environment readiness checks."
|
||||
fi
|
||||
elif [[ $i -lt $((NUM_TASKS - 1)) ]]; then
|
||||
sleep $STAGGER_DELAY
|
||||
fi
|
||||
done
|
||||
|
||||
echo "=============================================="
|
||||
echo "All tasks started in tmux session: $TMUX_SESSION"
|
||||
echo " - Attach: tmux attach -t $TMUX_SESSION"
|
||||
echo " - List: tmux list-windows -t $TMUX_SESSION"
|
||||
echo " - Select: tmux select-window -t $TMUX_SESSION:{window_name}"
|
||||
echo "Monitor: tail -f $JOB_DIR/*.log"
|
||||
echo "UI: streamlit run rdagent/app/finetune/llm/ui/app.py (Job Folder: $JOB_DIR)"
|
||||
@@ -0,0 +1,128 @@
|
||||
{
|
||||
"_comment": "Benchmark scenarios for FT tasks. Used by run_ft_job.sh and UI.",
|
||||
|
||||
"aime24": {
|
||||
"category": "math",
|
||||
"scenario": "Improve the model's ability to solve advanced competition math problems through multi-step reasoning, including number theory, combinatorics, geometry, and algebraic manipulation, with answers expressed as integers from 0 to 999.",
|
||||
"benchmark_description": "AIME 2024 (American Invitational Mathematics Examination) - Advanced high school math competition problems requiring creative problem-solving. Each answer is an integer 0-999. Topics include number theory, algebra, geometry, trigonometry, probability, and combinatorics. Problems require multi-step reasoning and often have elegant solutions. Expected Output Format: Put final answer within \\boxed{}, e.g., \\boxed{42}."
|
||||
},
|
||||
"aime25": {
|
||||
"category": "math",
|
||||
"scenario": "Improve the model's ability to solve advanced competition math problems through multi-step reasoning, including number theory, combinatorics, geometry, and algebraic manipulation, with answers expressed as integers from 0 to 999.",
|
||||
"benchmark_description": "AIME 2025 (American Invitational Mathematics Examination) - Advanced high school math competition problems requiring creative problem-solving. Each answer is an integer 0-999. Topics include number theory, algebra, geometry, trigonometry, probability, and combinatorics. Problems require multi-step reasoning and often have elegant solutions. Expected Output Format: Put final answer within \\boxed{}, e.g., \\boxed{42}."
|
||||
},
|
||||
"panorama": {
|
||||
"category": "patent",
|
||||
"scenario": "Improve the model's patent examination capabilities including prior art retrieval, paragraph identification, and novelty/obviousness classification based on USPTO standards.",
|
||||
"benchmark_description": "PANORAMA tests patent examination capabilities based on real USPTO Office Actions. Tasks include: retrieving relevant prior art patents, identifying specific paragraphs in prior art that relate to claims, and classifying claims as allowable, lacking novelty (102), or obvious (103). Requires understanding patent law, technical document analysis, and legal reasoning. Expected Output Format: Return JSON with task-specific format (see subtask descriptions)."
|
||||
},
|
||||
"panorama_par4pc": {
|
||||
"category": "patent",
|
||||
"scenario": "Improve the model's ability to retrieve relevant prior art patents given a patent claim, by understanding claim scope, identifying technical similarities, and ranking patents by relevance for rejection analysis.",
|
||||
"benchmark_description": "PAR4PC (Prior Art Retrieval for Patent Claims) - Given a patent claim, retrieve the most relevant prior art patents from a candidate pool. Requires understanding claim scope, identifying technical similarities, and ranking patents by relevance for potential 35 USC 102/103 rejections. Expected Output Format: Return JSON: {\"answer\": \"A\"} for single patent or {\"answer\": [\"A\", \"C\"]} for multiple patents (codes A-H)."
|
||||
},
|
||||
"panorama_pi4pc": {
|
||||
"category": "patent",
|
||||
"scenario": "Improve the model's ability to identify specific paragraphs in prior art patents that are most relevant for evaluating a claim's novelty and obviousness through element-by-element analysis.",
|
||||
"benchmark_description": "PI4PC (Paragraph Identification for Patent Claims) - Given a patent claim and cited prior art patent, identify specific paragraphs most relevant for evaluating novelty and obviousness. Requires detailed technical reading, element-by-element claim analysis, and understanding how prior art teachings map to claim limitations. Expected Output Format: Return JSON: {\"answer\": \"<paragraph_id>\"}."
|
||||
},
|
||||
"panorama_noc4pc": {
|
||||
"category": "patent",
|
||||
"scenario": "Improve the model's ability to classify patent claims as allowable, anticipated, or obvious by applying patent law standards to analyze claim limitations against prior art.",
|
||||
"benchmark_description": "NOC4PC (Novelty/Obviousness Classification) - Classify patent claims as ALLOW (patentable), 102 (anticipated/lacks novelty), or 103 (obvious). Requires applying patent law standards: 102 when single reference discloses all elements, 103 when combination of references with motivation makes claim obvious to skilled artisan. Expected Output Format: Return JSON: {\"code\": \"ALLOW\"} or {\"code\": \"102\"} or {\"code\": \"103\"}."
|
||||
},
|
||||
"panorama_par4pc_cot": {
|
||||
"category": "patent",
|
||||
"scenario": "Improve the model's ability to retrieve relevant prior art patents while providing explicit chain-of-thought reasoning explaining which claim elements each patent teaches and how it supports a rejection.",
|
||||
"benchmark_description": "PAR4PC with chain-of-thought - Retrieve relevant prior art while providing explicit reasoning. Explain why each retrieved patent is relevant: which claim elements it teaches, what technical problems it addresses, and how it could support a rejection. Expected Output Format: Provide reasoning first, then return JSON: {\"answer\": \"A\"} or {\"answer\": [\"A\", \"C\"]}."
|
||||
},
|
||||
"panorama_pi4pc_cot": {
|
||||
"category": "patent",
|
||||
"scenario": "Improve the model's ability to identify relevant prior art paragraphs while providing element-by-element mapping showing how specific paragraph teachings correspond to claim limitations.",
|
||||
"benchmark_description": "PI4PC with chain-of-thought - Identify relevant prior art paragraphs while explaining the technical connections. Provide element-by-element mapping showing how specific paragraph teachings correspond to claim limitations. Expected Output Format: Provide reasoning first, then return JSON: {\"answer\": \"<paragraph_id>\"}."
|
||||
},
|
||||
"panorama_noc4pc_cot": {
|
||||
"category": "patent",
|
||||
"scenario": "Improve the model's ability to classify patent claims with examiner-style rationale, explaining how references anticipate limitations or how combinations with motivation render claims obvious.",
|
||||
"benchmark_description": "NOC4PC with chain-of-thought - Classify claims with examiner-style rationale. For 102: explain how reference anticipates each limitation. For 103: identify references, explain motivation to combine, and show how combination renders claim obvious. Use proper USPTO citation format. Expected Output Format: Return JSON: {\"reason\": \"<Office Action analysis>\", \"code\": \"ALLOW\"|\"102\"|\"103\"}."
|
||||
},
|
||||
|
||||
"chemcotbench": {
|
||||
"category": "chemistry",
|
||||
"scenario": "Improve the model's chemical reasoning capabilities on molecular structures including understanding molecular features, editing molecules, optimizing properties, and predicting reaction outcomes.",
|
||||
"benchmark_description": "ChemCoTBench tests step-wise chemical reasoning on SMILES molecular structures. Tasks include molecule understanding (identify functional groups, ring systems), molecule editing (add/delete/substitute groups while maintaining validity), molecule optimization (modify for desired properties), and reaction prediction (products, mechanisms, conditions). Contains subtasks with different output requirements. Expected Output Format: Return JSON: {\"output\": \"<answer>\"} where answer format depends on subtask - SMILES string for molecular tasks, numeric count for counting tasks, or Yes/No for equivalence tasks."
|
||||
},
|
||||
"chemcotbench_mol_und": {
|
||||
"category": "chemistry",
|
||||
"scenario": "Improve the model's ability to analyze molecular structures and identify structural features including functional groups (hydroxyl, carboxyl, amine), ring systems (aromatic, aliphatic), and molecular scaffolds.",
|
||||
"benchmark_description": "Molecule Understanding - Analyze SMILES strings for structural features. Subtasks: (1) fg_count/ring_count: return integer count, (2) equivalence/ring_system_scaffold: return Yes or No, (3) Murcko_scaffold: return SMILES string. Requires parsing SMILES notation and applying organic chemistry knowledge. Expected Output Format: Return JSON: {\"output\": \"<answer>\"} where answer is integer/Yes/No/SMILES depending on subtask."
|
||||
},
|
||||
"chemcotbench_mol_edit": {
|
||||
"category": "chemistry",
|
||||
"scenario": "Improve the model's ability to perform precise structural modifications on molecules (add, delete, substitute functional groups) while maintaining chemical validity and molecule integrity.",
|
||||
"benchmark_description": "Molecule Editing - Perform structural modifications on SMILES. Subtasks: add (add functional group), delete (remove group), sub (substitute group). Output must be valid SMILES representing chemically feasible molecules. Expected Output Format: Return JSON: {\"output\": \"<valid SMILES>\"}. SMILES validity is verified using RDKit."
|
||||
},
|
||||
"chemcotbench_mol_opt": {
|
||||
"category": "chemistry",
|
||||
"scenario": "Improve the model's ability to modify molecular structures to achieve target properties such as improved solubility, drug-likeness, or binding affinity to specific biological targets.",
|
||||
"benchmark_description": "Molecule Optimization - Modify structures to achieve target properties. Subtasks: drd/gsk/jnk (binding affinity to DRD2/GSK3β/JNK3 targets), logp (lipophilicity), qed (drug-likeness), solubility. Requires understanding structure-property relationships. Expected Output Format: Return JSON: {\"output\": \"<optimized SMILES>\"}."
|
||||
},
|
||||
"chemcotbench_reaction": {
|
||||
"category": "chemistry",
|
||||
"scenario": "Improve the model's ability to predict chemical reaction outcomes including forward synthesis, retrosynthesis, mechanism selection, and reaction conditions based on functional group transformations.",
|
||||
"benchmark_description": "Reaction Prediction - Predict reaction outcomes. Subtasks: fs (forward synthesis: reactants→products), retro (retrosynthesis: products→reactants), rcr (reaction condition recommendation), nepp (named reaction prediction), mechsel (mechanism selection). Requires understanding reaction types and functional group transformations. Expected Output Format: Return JSON: {\"output\": \"<SMILES or text answer>\"}."
|
||||
},
|
||||
|
||||
"tablebench_data_analysis": {
|
||||
"category": "table_qa",
|
||||
"scenario": "Improve the model's ability to analyze tabular data for complex questions including trend identification, correlation analysis, statistical computation, and data-driven forecasting.",
|
||||
"benchmark_description": "Table Data Analysis - Analyze tabular data to answer complex questions. Subtask types with different evaluation: (1) CorrelationAnalysis/TrendForecasting/StatisticalAnalysis: numeric answers with ±10% relative error tolerance, (2) ImpactAnalysis: exact match required, (3) Other analysis types: evaluated using ROUGE-L. Requires reading tables accurately and applying analytical reasoning. Expected Output Format: End response with \"Final Answer: <value>\"."
|
||||
},
|
||||
"tablebench_fact_checking": {
|
||||
"category": "table_qa",
|
||||
"scenario": "Improve the model's ability to verify factual claims against tabular data through accurate data extraction, implicit relationship understanding, and multi-hop reasoning across table cells.",
|
||||
"benchmark_description": "Table Fact Checking - Answer table-based factual questions accurately. Questions may ask for specific information (numbers, names, dates) or verification (Yes/No, True/False). Uses Exact Match evaluation. Expected Output Format: End response with \"Final Answer: <value>\" where value is the precise answer to the question."
|
||||
},
|
||||
"tablebench_numerical_reasoning": {
|
||||
"category": "table_qa",
|
||||
"scenario": "Improve the model's ability to perform mathematical operations on table data including arithmetic, aggregations (sum, average, count), comparisons, percentages, and multi-step calculations.",
|
||||
"benchmark_description": "Table Numerical Reasoning - Perform mathematical operations on table data: arithmetic (sum, difference, product), aggregations (average, count, max/min), comparisons, percentages, and multi-step calculations. Requires accurate number extraction and correct mathematical computation. Expected Output Format: End response with \"Final Answer: <numeric value>\"."
|
||||
},
|
||||
"tablebench_visualization": {
|
||||
"category": "table_qa",
|
||||
"scenario": "Improve the model's ability to generate Python code that creates appropriate visualizations (bar, line, pie, scatter charts) from tabular data with correct chart type selection and data mapping.",
|
||||
"benchmark_description": "Table Visualization - Generate Python code to create appropriate visualizations from tabular data: bar charts, line charts, pie charts, scatter plots. Select correct chart type for data, map columns correctly to axes, and produce executable matplotlib/pandas code. Expected Output Format: Return Python code in ```python code block using matplotlib/pandas. Code will be executed to verify correctness."
|
||||
},
|
||||
"tablebench_gen": {
|
||||
"category": "table_qa",
|
||||
"scenario": "Improve the model's overall table question answering capabilities across fact checking, numerical reasoning, data analysis, and visualization by understanding table structure and generating accurate answers.",
|
||||
"benchmark_description": "TableBench General - Comprehensive table QA covering fact checking, numerical reasoning, data analysis, and visualization. Questions require understanding table structure, extracting relevant data, performing reasoning or computation, and generating accurate answers or code. Expected Output Format: End response with \"Final Answer: <answer>\"."
|
||||
},
|
||||
|
||||
"FinanceIQ_gen": {
|
||||
"category": "finance",
|
||||
"scenario": "Improve the model's financial domain knowledge and reasoning capabilities across Chinese financial certification exams including CPA, banking, securities, fund, futures, insurance, tax, and actuarial qualifications through multiple-choice question answering.",
|
||||
"benchmark_description": "FinanceIQ tests financial domain knowledge through multiple-choice questions (A/B/C/D). Covers 10 Chinese financial certification exams: CPA (注册会计师), banking qualification, securities qualification, fund qualification, futures qualification, insurance qualification (CICE), tax advisor, economist, financial planner, and actuary. Uses LLM Judge for evaluation with 5-shot in-context learning. Evaluation metric: accuracy."
|
||||
},
|
||||
|
||||
"bioprobench_gen": {
|
||||
"category": "biology",
|
||||
"scenario": "Improve the model's ability to generate complete, detailed experimental protocol steps from research context, including specific reagent concentrations, temperatures, incubation times, and equipment settings.",
|
||||
"benchmark_description": "Protocol Generation - Generate complete experimental protocol steps given research context and objectives. Output detailed, actionable instructions: specify reagent concentrations, temperatures, incubation times, equipment settings. Protocols must be scientifically valid and reproducible. Expected Output Format: Wrap protocol steps in [ANSWER_START]Step 1: ... Step 2: ...[ANSWER_END]. Evaluated using BLEU, ROUGE, and step matching metrics."
|
||||
},
|
||||
"bioprobench_ord": {
|
||||
"category": "biology",
|
||||
"scenario": "Improve the model's ability to arrange shuffled experimental steps in correct sequence. Output MUST be a valid Python list format: [ANSWER_START][0, 2, 1, 3][ANSWER_END]. Use brackets and commas, NOT space-separated indices.",
|
||||
"benchmark_description": "Step Ordering - Arrange shuffled experimental procedure steps in correct logical and temporal sequence. Requires understanding procedural dependencies: which steps must precede others, timing constraints, and scientific logic of experimental workflows. CRITICAL OUTPUT FORMAT: Answer MUST be a valid Python list with brackets and commas, e.g., [ANSWER_START][2, 0, 1, 3][ANSWER_END]. NOT space-separated (0 2 1 3 is WRONG), NOT without brackets (0, 2, 1, 3 is WRONG). Evaluated using Exact Match and Kendall's Tau."
|
||||
},
|
||||
"bioprobench_err": {
|
||||
"category": "biology",
|
||||
"scenario": "Improve the model's ability to identify errors in biological protocol text. CRITICAL SEMANTICS: True = step is CORRECT (no errors), False = step HAS ERRORS. This matches the benchmark prompt: 'If you find anything wrong, answer False.' Output format: [ANSWER_START]True or False[ANSWER_END].",
|
||||
"benchmark_description": "Error Correction - Identify errors in biological protocol text including incorrect temperatures, concentrations, reagents, or procedural mistakes. CRITICAL: The benchmark expects True if the protocol step is CORRECT (no errors), and False if it HAS ERRORS. This follows the prompt: 'If you find anything wrong, answer False.' Do NOT invert this logic. Expected Output Format: [ANSWER_START]True[ANSWER_END] for correct steps, [ANSWER_START]False[ANSWER_END] for erroneous steps."
|
||||
},
|
||||
"bioprobench_pqa": {
|
||||
"category": "biology",
|
||||
"scenario": "Improve the model's ability to extract specific factual information from experimental protocols including temperatures, concentrations, incubation times, reagent quantities, and procedural details.",
|
||||
"benchmark_description": "Protocol QA - Extract specific factual information from experimental protocols: temperatures, concentrations, incubation times, reagent quantities, equipment specifications, and procedural details. Requires careful reading and accurate information extraction from technical text. Expected Output Format: Return [ANSWER_START]<answer text> & <confidence 0-100>%[ANSWER_END], e.g., [ANSWER_START]Option A & 95%[ANSWER_END]. Evaluated using accuracy and Brier Score."
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"tasks": [
|
||||
{
|
||||
"model": "Qwen/Qwen2.5-7B-Instruct",
|
||||
"benchmark": "aime25",
|
||||
"gpus": "0,1",
|
||||
"timeout": "12h"
|
||||
},
|
||||
{
|
||||
"model": "Qwen/Qwen2.5-7B-Instruct",
|
||||
"benchmark": "chemcotbench_mol_edit",
|
||||
"gpus": "2,3",
|
||||
"timeout": "12h"
|
||||
},
|
||||
{
|
||||
"model": "Qwen/Qwen2.5-7B-Instruct",
|
||||
"benchmark": "tablebench_visualization",
|
||||
"gpus": "4,5",
|
||||
"timeout": "12h",
|
||||
"benchmark_description": "Table Visualization - generate executable Python code to create the requested chart from tabular data. Expected Output Format: return Python code in a ```python code block using matplotlib or pandas."
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
"""
|
||||
LLM Fine-tuning Entry Point
|
||||
|
||||
Standard RDLoop entry point for LLM fine-tuning, consistent with data science implementation.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from typing import Optional, cast
|
||||
|
||||
import fire
|
||||
|
||||
from rdagent.app.finetune.llm.conf import FT_RD_SETTING
|
||||
from rdagent.log import rdagent_logger as logger
|
||||
from rdagent.scenarios.finetune.loop import LLMFinetuneRDLoop
|
||||
|
||||
|
||||
def main(
|
||||
path: Optional[str] = None,
|
||||
checkout: bool = True,
|
||||
user_target_scenario: Optional[str] = None,
|
||||
benchmark: Optional[str] = None,
|
||||
benchmark_description: Optional[str] = None,
|
||||
dataset: Optional[str] = None,
|
||||
base_model: Optional[str] = None,
|
||||
upper_data_size_limit: Optional[int] = None,
|
||||
step_n: Optional[int] = None,
|
||||
loop_n: Optional[int] = None,
|
||||
timeout: Optional[str] = None,
|
||||
):
|
||||
"""
|
||||
LLM fine-tuning entry point
|
||||
|
||||
Parameters
|
||||
----------
|
||||
path :
|
||||
A path like `$LOG_PATH/__session__/1/0_propose`. This indicates that we restore the state after finishing step 0 in loop 1.
|
||||
checkout :
|
||||
Used to control the log session path. Boolean type, default is True.
|
||||
- If True, the new loop will use the existing folder and clear logs for sessions after the one corresponding to the given path.
|
||||
- If False, the new loop will use the existing folder but keep the logs for sessions after the one corresponding to the given path.
|
||||
dataset : str
|
||||
Dataset name for fine-tuning (e.g., 'shibing624/alpaca-zh')
|
||||
base_model : str, optional
|
||||
Model name for fine-tuning (e.g., 'Qwen/Qwen2.5-1.5B-Instruct').
|
||||
If not provided, auto-selects optimal model based on hardware and dataset.
|
||||
step_n : int, optional
|
||||
Number of steps to run; if None, runs indefinitely until completion or error
|
||||
loop_n : int, optional
|
||||
Number of loops to run; if None, runs indefinitely until completion or error
|
||||
timeout : str, optional
|
||||
Maximum duration for the entire process
|
||||
|
||||
Examples:
|
||||
.. code-block:: bash
|
||||
dotenv run -- python rdagent/app/finetune/llm/loop.py --dataset shibing624/alpaca-zh --base-model Qwen/Qwen2.5-1.5B-Instruct
|
||||
dotenv run -- python rdagent/app/finetune/llm/loop.py --dataset shibing624/alpaca-zh # TODO: not enabled yet
|
||||
"""
|
||||
|
||||
if user_target_scenario:
|
||||
FT_RD_SETTING.user_target_scenario = user_target_scenario
|
||||
assert (
|
||||
FT_RD_SETTING.user_target_scenario is None
|
||||
), "user_target_scenario is not yet supported, please specify via benchmark and benchmark_description"
|
||||
if upper_data_size_limit:
|
||||
FT_RD_SETTING.upper_data_size_limit = upper_data_size_limit
|
||||
logger.info(f"Set upper_data_size_limit to {FT_RD_SETTING.upper_data_size_limit}")
|
||||
if benchmark:
|
||||
FT_RD_SETTING.target_benchmark = benchmark
|
||||
if benchmark_description:
|
||||
FT_RD_SETTING.benchmark_description = benchmark_description
|
||||
assert FT_RD_SETTING.user_target_scenario or (
|
||||
FT_RD_SETTING.target_benchmark and FT_RD_SETTING.benchmark_description
|
||||
), "Either user_target_scenario or target_benchmark must be specified for LLM fine-tuning."
|
||||
|
||||
# Update configuration with provided parameters
|
||||
if dataset:
|
||||
FT_RD_SETTING.dataset = dataset
|
||||
if base_model:
|
||||
FT_RD_SETTING.base_model = base_model
|
||||
|
||||
# Create and run LLM fine-tuning loop
|
||||
data_set_target = FT_RD_SETTING.dataset if FT_RD_SETTING.dataset else "auto generated dataset"
|
||||
model_target = FT_RD_SETTING.base_model if FT_RD_SETTING.base_model else "auto selected model"
|
||||
|
||||
# Temporary assertion until auto-selection is implemented
|
||||
assert (
|
||||
FT_RD_SETTING.base_model is not None
|
||||
), "Base model auto selection not yet supported, please specify via --base-model"
|
||||
|
||||
logger.info(f"Starting LLM fine-tuning on dataset='{data_set_target}' with model='{model_target}'")
|
||||
|
||||
if path is None:
|
||||
loop = LLMFinetuneRDLoop(FT_RD_SETTING)
|
||||
else:
|
||||
loop = cast(LLMFinetuneRDLoop, LLMFinetuneRDLoop.load(str(path), checkout=checkout))
|
||||
|
||||
asyncio.run(loop.run(step_n=step_n, loop_n=loop_n, all_duration=timeout))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
fire.Fire(main)
|
||||
@@ -0,0 +1 @@
|
||||
# FT (Fine-tune) scenario UI
|
||||
@@ -0,0 +1,207 @@
|
||||
"""
|
||||
FT (Fine-tune) Timeline Viewer
|
||||
Hierarchical view: Session > Loop > Stage > EvoLoop > Events
|
||||
|
||||
Run:
|
||||
streamlit run rdagent/app/finetune/llm/ui/app.py
|
||||
"""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import streamlit as st
|
||||
from streamlit import session_state as state
|
||||
|
||||
from rdagent.app.finetune.llm.ui.benchmarks import get_core_metric_score
|
||||
from rdagent.app.finetune.llm.ui.components import render_session, render_summary
|
||||
from rdagent.app.finetune.llm.ui.config import ALWAYS_VISIBLE_TYPES, OPTIONAL_TYPES
|
||||
from rdagent.app.finetune.llm.ui.data_loader import (
|
||||
get_summary,
|
||||
get_valid_sessions,
|
||||
load_ft_session,
|
||||
)
|
||||
from rdagent.app.finetune.llm.ui.ft_summary import render_job_summary
|
||||
|
||||
DEFAULT_LOG_BASE = "log/"
|
||||
|
||||
|
||||
def get_job_options(base_path: Path) -> list[str]:
|
||||
"""
|
||||
Scan directory and return job options list.
|
||||
- "." means standalone tasks in root directory
|
||||
- Others are job directory names
|
||||
"""
|
||||
options = []
|
||||
has_root_tasks = False
|
||||
job_dirs = []
|
||||
|
||||
if not base_path.exists():
|
||||
return options
|
||||
|
||||
for d in base_path.iterdir():
|
||||
if not d.is_dir():
|
||||
continue
|
||||
# Check if standalone task (has __session__ directly)
|
||||
if (d / "__session__").exists():
|
||||
has_root_tasks = True
|
||||
# Check if job directory (subdirs have __session__)
|
||||
else:
|
||||
try:
|
||||
if any((sub / "__session__").exists() for sub in d.iterdir() if sub.is_dir()):
|
||||
job_dirs.append(d.name)
|
||||
except PermissionError:
|
||||
pass
|
||||
|
||||
# Sort job dirs by name descending (newest first, since names are date-based)
|
||||
job_dirs.sort(reverse=True)
|
||||
|
||||
# Add job dirs first, then root tasks at the end
|
||||
options.extend(job_dirs)
|
||||
if has_root_tasks:
|
||||
options.append(". (Current)")
|
||||
|
||||
return options
|
||||
|
||||
|
||||
def main():
|
||||
st.set_page_config(layout="wide", page_title="FT Timeline", page_icon="🔬")
|
||||
|
||||
# ========== Sidebar ==========
|
||||
with st.sidebar:
|
||||
# View mode selection
|
||||
view_mode = st.radio("View Mode", ["Job Summary", "Single Task"], horizontal=True)
|
||||
|
||||
st.divider()
|
||||
|
||||
default_log = os.environ.get("FT_LOG_PATH", DEFAULT_LOG_BASE)
|
||||
job_folder = default_log # Initialize for both modes
|
||||
selected_types = ALWAYS_VISIBLE_TYPES.copy() # Initialize for both modes
|
||||
is_root_job = False # Track if viewing root tasks
|
||||
|
||||
if view_mode == "Job Summary":
|
||||
# Job Summary mode
|
||||
st.header("Job")
|
||||
base_folder = st.text_input("Base Folder", value=default_log, key="base_folder_input")
|
||||
base_path = Path(base_folder)
|
||||
|
||||
job_options = get_job_options(base_path)
|
||||
if job_options:
|
||||
selected_job = st.selectbox("Select Job", job_options, key="job_select")
|
||||
if selected_job.startswith("."):
|
||||
job_folder = base_folder
|
||||
is_root_job = True
|
||||
else:
|
||||
job_folder = str(base_path / selected_job)
|
||||
# Save to session_state for Single Task mode
|
||||
state.selected_job_folder = job_folder
|
||||
else:
|
||||
st.warning("No jobs found in this directory")
|
||||
job_folder = base_folder
|
||||
|
||||
if st.button("Refresh", type="primary", key="refresh_job"):
|
||||
st.rerun()
|
||||
else:
|
||||
# Single Task mode
|
||||
st.header("Session")
|
||||
# Use job_folder from Job Summary mode if available
|
||||
default_path = getattr(state, "selected_job_folder", default_log)
|
||||
log_folder = st.text_input("Log Folder", value=default_path)
|
||||
log_path = Path(log_folder)
|
||||
|
||||
sessions = get_valid_sessions(log_path)
|
||||
if not sessions:
|
||||
st.warning("No valid sessions found")
|
||||
return
|
||||
|
||||
selected_session = st.selectbox("Session", sessions)
|
||||
|
||||
if st.button("Load", type="primary") or "session" not in state:
|
||||
with st.spinner("Loading..."):
|
||||
state.session = load_ft_session(log_path / selected_session)
|
||||
state.session_name = selected_session
|
||||
|
||||
st.divider()
|
||||
|
||||
# Optional type toggles
|
||||
st.subheader("Show More")
|
||||
selected_types = ALWAYS_VISIBLE_TYPES.copy()
|
||||
for event_type, (label, default) in OPTIONAL_TYPES.items():
|
||||
if st.toggle(label, value=default, key=f"toggle_{event_type}"):
|
||||
selected_types.append(event_type)
|
||||
|
||||
st.divider()
|
||||
|
||||
# Display options
|
||||
st.subheader("Display Options")
|
||||
state.render_markdown = st.toggle("Render Prompts", value=False, key="render_markdown_toggle")
|
||||
|
||||
st.divider()
|
||||
|
||||
# Summary in sidebar
|
||||
if "session" in state:
|
||||
summary = get_summary(state.session)
|
||||
st.subheader("Summary")
|
||||
st.metric("Loops", summary.get("loop_count", 0))
|
||||
st.metric("LLM Calls", summary.get("llm_call_count", 0))
|
||||
success = summary.get("docker_success", 0)
|
||||
fail = summary.get("docker_fail", 0)
|
||||
st.metric("Docker", f"{success}✓ / {fail}✗")
|
||||
|
||||
# ========== Main Content ==========
|
||||
if view_mode == "Job Summary":
|
||||
st.title("📊 FT Job Summary")
|
||||
job_path = Path(job_folder)
|
||||
if job_path.exists():
|
||||
render_job_summary(job_path, is_root=is_root_job)
|
||||
else:
|
||||
st.warning(f"Job folder not found: {job_folder}")
|
||||
return
|
||||
|
||||
# Single Task mode
|
||||
st.title("🔬 FT Timeline Viewer")
|
||||
|
||||
if "session" not in state:
|
||||
st.info("Select a session and click **Load** to view")
|
||||
return
|
||||
|
||||
session = state.session
|
||||
summary = get_summary(session)
|
||||
|
||||
# Global info header (Base Model, Datasets, Benchmark) - compact style
|
||||
scenario_event = next((e for e in session.init_events if e.type == "scenario"), None)
|
||||
dataset_event = next((e for e in session.init_events if e.type == "dataset_selection"), None)
|
||||
|
||||
if scenario_event or dataset_event:
|
||||
if scenario_event and hasattr(scenario_event.content, "base_model"):
|
||||
st.markdown(f"🧠 **Model:** `{scenario_event.content.base_model}`")
|
||||
if dataset_event:
|
||||
selected = (
|
||||
dataset_event.content.get("selected_datasets", []) if isinstance(dataset_event.content, dict) else []
|
||||
)
|
||||
if selected:
|
||||
st.markdown(f"📂 **Datasets:** `{', '.join(selected)}`")
|
||||
if scenario_event and hasattr(scenario_event.content, "target_benchmark"):
|
||||
st.markdown(f"🎯 **Benchmark:** `{scenario_event.content.target_benchmark}`")
|
||||
# Display baseline benchmark score
|
||||
if scenario_event and hasattr(scenario_event.content, "baseline_benchmark_score"):
|
||||
baseline = scenario_event.content.baseline_benchmark_score
|
||||
if baseline and isinstance(baseline, dict):
|
||||
benchmark_name = getattr(scenario_event.content, "target_benchmark", "")
|
||||
accuracy_summary = baseline.get("accuracy_summary", {})
|
||||
if accuracy_summary:
|
||||
result = get_core_metric_score(benchmark_name, accuracy_summary)
|
||||
if result:
|
||||
metric_name, score, _ = result
|
||||
st.markdown(f"📊 **Baseline:** `{metric_name} = {score:.1f}`")
|
||||
|
||||
# Summary bar
|
||||
render_summary(summary)
|
||||
|
||||
st.divider()
|
||||
|
||||
# Hierarchical view
|
||||
render_session(session, selected_types)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,70 @@
|
||||
"""Benchmark processors for core metric extraction.
|
||||
|
||||
Each benchmark has its own processor that knows how to extract
|
||||
the core metric name and value from accuracy_summary data.
|
||||
"""
|
||||
|
||||
from .bioprobench import BioProBenchProcessor
|
||||
from .chemcotbench import ChemCotBenchProcessor
|
||||
from .financeiq import FinanceIQProcessor
|
||||
from .panorama import PanoramaProcessor
|
||||
from .tablebench import TableBenchProcessor
|
||||
|
||||
PROCESSORS = [
|
||||
FinanceIQProcessor,
|
||||
PanoramaProcessor,
|
||||
ChemCotBenchProcessor,
|
||||
TableBenchProcessor,
|
||||
BioProBenchProcessor,
|
||||
]
|
||||
|
||||
|
||||
def get_core_metric_score(benchmark_name: str, accuracy_summary: dict) -> tuple[str, float, bool] | None:
|
||||
"""Get core metric name, score, and direction for a benchmark.
|
||||
|
||||
Args:
|
||||
benchmark_name: The benchmark name (e.g., "FinanceIQ", "panorama_par4pc")
|
||||
accuracy_summary: {dataset_name: {metric: value, ...}, ...}
|
||||
|
||||
Returns:
|
||||
(metric_name, value, higher_is_better) or None
|
||||
- metric_name: includes "(average)" suffix if multiple datasets are averaged
|
||||
- value: the score
|
||||
- higher_is_better: True if higher values are better (use ↑), False otherwise (use ↓)
|
||||
"""
|
||||
for processor in PROCESSORS:
|
||||
if processor.match(benchmark_name):
|
||||
return processor.get_core_metric(accuracy_summary)
|
||||
|
||||
# Default fallback: use first numeric value with "accuracy" label
|
||||
scores = []
|
||||
for ds, metrics in accuracy_summary.items():
|
||||
if not isinstance(metrics, dict):
|
||||
continue
|
||||
if "accuracy" in metrics:
|
||||
scores.append(float(metrics["accuracy"]))
|
||||
else:
|
||||
for v in metrics.values():
|
||||
if isinstance(v, (int, float)):
|
||||
scores.append(float(v))
|
||||
break
|
||||
|
||||
if not scores:
|
||||
return None
|
||||
|
||||
avg = sum(scores) / len(scores)
|
||||
if len(scores) == 1:
|
||||
return ("accuracy", avg, True) # higher is better
|
||||
else:
|
||||
return ("accuracy (average)", avg, True) # higher is better
|
||||
|
||||
|
||||
__all__ = [
|
||||
"get_core_metric_score",
|
||||
"PROCESSORS",
|
||||
"FinanceIQProcessor",
|
||||
"PanoramaProcessor",
|
||||
"ChemCotBenchProcessor",
|
||||
"TableBenchProcessor",
|
||||
"BioProBenchProcessor",
|
||||
]
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Base class for benchmark core metric extraction."""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
|
||||
class BenchmarkProcessor(ABC):
|
||||
"""Base class for benchmark core metric extraction."""
|
||||
|
||||
# Metrics where higher values are better (default assumption)
|
||||
# Override in subclass if needed
|
||||
HIGHER_IS_BETTER: set[str] = {
|
||||
"accuracy",
|
||||
"exact_match",
|
||||
"f1",
|
||||
"f1_score",
|
||||
"macro_f1",
|
||||
"correct_rate",
|
||||
"success_rate",
|
||||
"gold_hit_rate",
|
||||
"score",
|
||||
"scaffold_hard",
|
||||
"kendall_tau",
|
||||
"ROUGE-L",
|
||||
}
|
||||
|
||||
@classmethod
|
||||
@abstractmethod
|
||||
def match(cls, benchmark_name: str) -> bool:
|
||||
"""Check if this processor handles the given benchmark."""
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
@abstractmethod
|
||||
def get_core_metric(cls, accuracy_summary: dict) -> tuple[str, float, bool] | None:
|
||||
"""Extract core metric name, value, and direction from accuracy_summary.
|
||||
|
||||
Args:
|
||||
accuracy_summary: {dataset_name: {metric: value, ...}, ...}
|
||||
|
||||
Returns:
|
||||
(metric_name, value, higher_is_better) or None
|
||||
- metric_name: includes "(average)" suffix if multiple datasets
|
||||
- value: the score
|
||||
- higher_is_better: True if higher values are better, False otherwise
|
||||
"""
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def is_higher_better(cls, metric_name: str) -> bool:
|
||||
"""Check if higher values are better for this metric."""
|
||||
# Remove (average) suffix for checking
|
||||
base_metric = metric_name.replace(" (average)", "").strip()
|
||||
return base_metric.lower() in {m.lower() for m in cls.HIGHER_IS_BETTER}
|
||||
@@ -0,0 +1,60 @@
|
||||
"""BioProBench benchmark processor."""
|
||||
|
||||
from .base import BenchmarkProcessor
|
||||
|
||||
|
||||
class BioProBenchProcessor(BenchmarkProcessor):
|
||||
"""BioProBench: Biology protocol benchmark with different task types."""
|
||||
|
||||
CORE_METRICS = {
|
||||
"pqa": "accuracy",
|
||||
"ord": "kendall_tau",
|
||||
"err": "f1",
|
||||
"gen": "ROUGE-L",
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def match(cls, benchmark_name: str) -> bool:
|
||||
return "bioprobench" in benchmark_name.lower()
|
||||
|
||||
@classmethod
|
||||
def get_core_metric(cls, accuracy_summary: dict) -> tuple[str, float, bool] | None:
|
||||
scores = []
|
||||
metrics_used = []
|
||||
|
||||
for ds, metrics in accuracy_summary.items():
|
||||
if not isinstance(metrics, dict):
|
||||
continue
|
||||
ds_lower = ds.lower()
|
||||
# Find matching core metric
|
||||
core_metric = "accuracy" # fallback
|
||||
for pattern, metric in cls.CORE_METRICS.items():
|
||||
if pattern in ds_lower:
|
||||
core_metric = metric
|
||||
break
|
||||
|
||||
if core_metric in metrics:
|
||||
scores.append(float(metrics[core_metric]))
|
||||
metrics_used.append(core_metric)
|
||||
elif core_metric.lower() in [k.lower() for k in metrics.keys()]:
|
||||
# Case-insensitive fallback for metrics like "ROUGE-L"
|
||||
for k, v in metrics.items():
|
||||
if k.lower() == core_metric.lower():
|
||||
scores.append(float(v))
|
||||
metrics_used.append(core_metric)
|
||||
break
|
||||
|
||||
if not scores:
|
||||
return None
|
||||
|
||||
avg = sum(scores) / len(scores)
|
||||
unique = list(set(metrics_used))
|
||||
|
||||
if len(scores) == 1:
|
||||
metric_name = unique[0]
|
||||
elif len(unique) == 1:
|
||||
metric_name = f"{unique[0]} (average)"
|
||||
else:
|
||||
metric_name = "mixed (average)"
|
||||
|
||||
return (metric_name, avg, cls.is_higher_better(metric_name))
|
||||
@@ -0,0 +1,105 @@
|
||||
"""ChemCotBench benchmark processor."""
|
||||
|
||||
from .base import BenchmarkProcessor
|
||||
|
||||
|
||||
class ChemCotBenchProcessor(BenchmarkProcessor):
|
||||
"""ChemCotBench: Chemistry reasoning with various subtasks.
|
||||
|
||||
All metrics are 0-100 percentages, enabling unified averaging within each subset.
|
||||
"""
|
||||
|
||||
# Define core metric field names for each task
|
||||
CORE_METRICS = {
|
||||
# Molecular understanding
|
||||
"mol_und_fg_count": "accuracy",
|
||||
"mol_und_ring_count": "accuracy",
|
||||
"mol_und_murcko_scaffold": "scaffold_hard", # Exact match rate (0-100%)
|
||||
"mol_und_ring_system_scaffold": "score", # "Yes" ratio (0-100%)
|
||||
"mol_und_equivalence": "accuracy",
|
||||
# Molecular editing
|
||||
"mol_edit_add": "correct_rate",
|
||||
"mol_edit_delete": "correct_rate",
|
||||
"mol_edit_sub": "correct_rate",
|
||||
# Molecular optimization (prefix match)
|
||||
"mol_opt_": "success_rate",
|
||||
# Reaction tasks - unified to exact_match
|
||||
"reaction_fs": "exact_match",
|
||||
"reaction_retro": "exact_match",
|
||||
"reaction_nepp": "exact_match",
|
||||
"reaction_rcr": "exact_match",
|
||||
"reaction_mechsel": "exact_match", # Will fallback to accuracy if exact_match not found
|
||||
}
|
||||
|
||||
# Metric groups: unified display names for each subset
|
||||
METRIC_GROUPS = {
|
||||
"mol_und": "accuracy", # mol_und subset displays as accuracy
|
||||
"mol_edit": "correct_rate",
|
||||
"mol_opt": "success_rate",
|
||||
"reaction": "exact_match", # reaction subset displays as exact_match
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def match(cls, benchmark_name: str) -> bool:
|
||||
return "chemcot" in benchmark_name.lower()
|
||||
|
||||
@classmethod
|
||||
def get_core_metric(cls, accuracy_summary: dict) -> tuple[str, float, bool] | None:
|
||||
scores = []
|
||||
group_detected = None
|
||||
|
||||
for ds, metrics in accuracy_summary.items():
|
||||
if not isinstance(metrics, dict):
|
||||
continue
|
||||
ds_lower = ds.lower()
|
||||
|
||||
# Detect subset type
|
||||
for group in cls.METRIC_GROUPS:
|
||||
if group in ds_lower:
|
||||
group_detected = group
|
||||
break
|
||||
|
||||
# Find matching core metric
|
||||
core_metric = "accuracy" # fallback
|
||||
for pattern, metric in cls.CORE_METRICS.items():
|
||||
# Prefix match for patterns ending with _
|
||||
if pattern.endswith("_"):
|
||||
if pattern in ds_lower:
|
||||
core_metric = metric
|
||||
break
|
||||
else:
|
||||
if pattern in ds_lower:
|
||||
core_metric = metric
|
||||
break
|
||||
|
||||
# Try to get metric value with fallback support
|
||||
value = None
|
||||
if core_metric in metrics:
|
||||
value = float(metrics[core_metric])
|
||||
elif core_metric == "exact_match" and "accuracy" in metrics:
|
||||
# reaction_mechsel fallback: exact_match -> accuracy
|
||||
value = float(metrics["accuracy"])
|
||||
|
||||
if value is not None:
|
||||
scores.append(value)
|
||||
|
||||
if not scores:
|
||||
return None
|
||||
|
||||
avg = sum(scores) / len(scores)
|
||||
|
||||
# Use unified metric name for the detected subset
|
||||
if group_detected and group_detected in cls.METRIC_GROUPS:
|
||||
unified_name = cls.METRIC_GROUPS[group_detected]
|
||||
if len(scores) == 1:
|
||||
metric_name = unified_name
|
||||
else:
|
||||
metric_name = f"{unified_name} (average)"
|
||||
else:
|
||||
# Fallback for unknown subsets
|
||||
if len(scores) == 1:
|
||||
metric_name = "accuracy"
|
||||
else:
|
||||
metric_name = "accuracy (average)"
|
||||
|
||||
return (metric_name, avg, cls.is_higher_better(metric_name))
|
||||
@@ -0,0 +1,29 @@
|
||||
"""FinanceIQ benchmark processor."""
|
||||
|
||||
from .base import BenchmarkProcessor
|
||||
|
||||
|
||||
class FinanceIQProcessor(BenchmarkProcessor):
|
||||
"""FinanceIQ: 10 exam subjects, all use accuracy."""
|
||||
|
||||
@classmethod
|
||||
def match(cls, benchmark_name: str) -> bool:
|
||||
return "financeiq" in benchmark_name.lower()
|
||||
|
||||
@classmethod
|
||||
def get_core_metric(cls, accuracy_summary: dict) -> tuple[str, float, bool] | None:
|
||||
scores = []
|
||||
for ds, metrics in accuracy_summary.items():
|
||||
if not isinstance(metrics, dict):
|
||||
continue
|
||||
if "accuracy" in metrics:
|
||||
scores.append(float(metrics["accuracy"]))
|
||||
|
||||
if not scores:
|
||||
return None
|
||||
|
||||
avg = sum(scores) / len(scores)
|
||||
if len(scores) == 1:
|
||||
return ("accuracy", avg, True) # higher is better
|
||||
else:
|
||||
return ("accuracy (average)", avg, True) # higher is better
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Panorama benchmark processor."""
|
||||
|
||||
from .base import BenchmarkProcessor
|
||||
|
||||
|
||||
class PanoramaProcessor(BenchmarkProcessor):
|
||||
"""Panorama: Different sub-datasets use different metrics."""
|
||||
|
||||
CORE_METRICS = {
|
||||
"par4pc": "macro_f1",
|
||||
"pi4pc": "gold_hit_rate",
|
||||
"noc4pc": "macro_f1",
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def match(cls, benchmark_name: str) -> bool:
|
||||
return "panorama" in benchmark_name.lower()
|
||||
|
||||
@classmethod
|
||||
def get_core_metric(cls, accuracy_summary: dict) -> tuple[str, float, bool] | None:
|
||||
scores = []
|
||||
metrics_used = []
|
||||
|
||||
for ds, metrics in accuracy_summary.items():
|
||||
if not isinstance(metrics, dict):
|
||||
continue
|
||||
ds_lower = ds.lower()
|
||||
# Find matching core metric
|
||||
core_metric = "accuracy" # fallback
|
||||
for pattern, metric in cls.CORE_METRICS.items():
|
||||
if pattern in ds_lower:
|
||||
core_metric = metric
|
||||
break
|
||||
|
||||
if core_metric in metrics:
|
||||
scores.append(float(metrics[core_metric]))
|
||||
metrics_used.append(core_metric)
|
||||
|
||||
if not scores:
|
||||
return None
|
||||
|
||||
avg = sum(scores) / len(scores)
|
||||
unique = list(set(metrics_used))
|
||||
|
||||
if len(scores) == 1:
|
||||
metric_name = unique[0]
|
||||
elif len(unique) == 1:
|
||||
metric_name = f"{unique[0]} (average)"
|
||||
else:
|
||||
metric_name = "mixed (average)"
|
||||
|
||||
return (metric_name, avg, cls.is_higher_better(metric_name))
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user