# Disabled LLM CI workflow - manually enabled when needed name: LLM CI (DISABLED) #on: # pull_request: # types: [opened, synchronize, reopened] # branches: [develop] # schedule: # - cron: "2 0 * * *" # workflow_call: # inputs: # run_downstream: # required: true # type: string # image_name: # required: true # type: string #concurrency: # group: ${{ github.workflow }}-${{ github.event.pull_request.number }} # cancel-in-progress: true #env: # PR_ID: ${{ github.event.pull_request.number }} COMMIT_ID: ${{ github.event.pull_request.head.sha }} TASK: paddlenlp-CI-${{ github.event.pull_request.number }}-llm ci_scripts: /workspace/PaddleNLP/scripts/regression BRANCH: ${{ github.event.pull_request.base.ref }} AGILE_COMPILE_BRANCH: ${{ github.event.pull_request.base.ref }} CI_name: llm-ci no_proxy: "localhost,bj.bcebos.com,su.bcebos.com,bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn" HF_ENDPOINT: https://hf-mirror.com STUDIO_GIT_HOST: http://git.prod.idc-to-cloud.aistudio.baidu-int.com PPNLP_HOME: /ssd1/paddlenlp HF_DATASETS_CACHE: /ssd1/paddlenlp/huggingface/datasets TRANSFORMERS_CACHE: /ssd1/paddlenlp/huggingface CCACHE_DIR: /home/data/gzcfs/.ccache/gpubox RUN_DOWNSTREAM: ${{ inputs.run_downstream }} #defaults: # run: # shell: bash #jobs: # llm-ci: # name: llm-ci (DISABLED) runs-on: [self-hosted, ernie-8gpu] steps: - name: Determine Image Name env: IMAGE_NAME: ${{ inputs.image_name }} run: | if [[ -n "${IMAGE_NAME}" ]]; then echo "IMAGE_NAME=${IMAGE_NAME}" >> "$GITHUB_ENV" else echo "IMAGE_NAME=iregistry.baidu-int.com/paddlecloud/base-images:paddlecloud-ubuntu18.04-gcc8.2-cuda11.8-cudnn8.6-nccl2.15.5-paddlenlp-latest" >> "$GITHUB_ENV" fi - name: Run Container env: work_dir: ${{ github.workspace }} CACHE_DIR: /home/data/cfs/.cache FLAGS_dynamic_static_unified_comm: "True" python_version: "3.10" paddle_whl: https://paddle-qa.bj.bcebos.com/paddle-pipeline/Develop-GpuSome-LinuxCentos-Gcc82-Cuda118-Cudnn86-Trt85-Py310-CINN-Compile/latest/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl run: | container_name=${TASK}-$(date +%Y%m%d-%H%M%S) echo "container_name=${container_name}" >> "$GITHUB_ENV" if [[ "$RUN_DOWNSTREAM" == "false" ]]; then echo "Not in a pull_request or test_build event. Skipping..." else docker run -d -t --name ${container_name} --net=host -v /dev/shm:/dev/shm --shm-size=32G \ -v $work_dir/../../..:$work_dir/../../.. \ -v $work_dir:/workspace \ -v /home/.cache/pip:/home/.cache/pip \ -v /ssd1/paddlenlp:/ssd1/paddlenlp \ -v /home/data/gzcfs/.ccache/gpubox:/home/data/gzcfs/.ccache/gpubox \ -e BRANCH \ -e AGILE_COMPILE_BRANCH \ -e PR_ID \ -e COMMIT_ID \ -e work_dir \ -e ci_scripts \ -e no_proxy \ -e CI_name \ -e paddle_whl \ -e HF_ENDPOINT \ -e STUDIO_GIT_HOST \ -e PPNLP_HOME \ -e HF_DATASETS_CACHE \ -e TRANSFORMERS_CACHE \ -e CACHE_DIR \ -e FLAGS_dynamic_static_unified_comm \ -e python_version \ -w /workspace --runtime=nvidia $IMAGE_NAME fi - name: Download Code env: work_dir: ${{ github.workspace }} run: | if [[ "$RUN_DOWNSTREAM" == "false" ]]; then echo "Not in a pull_request or test_build event. Skipping.." else docker exec -t $container_name /bin/bash -c ' rm -rf * .[^.]* echo "Downloading PaddleNLP.tar.gz" wget -q --no-proxy https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddleNLP.tar --no-check-certificate echo "Extracting PaddleNLP.tar.gz" tar xf PaddleNLP.tar && rm -rf PaddleNLP.tar source $work_dir/../../../proxy cd PaddleNLP git config --global user.name "PaddleCI" git config --global user.email "paddle_ci@example.com" git pull git submodule update --init --recursive --force if [ -n "${PR_ID}" ]; then git fetch origin pull/${PR_ID}/head git checkout -b PR_${PR_ID} FETCH_HEAD git remote add upstream https://github.com/PaddlePaddle/PaddleNLP.git git fetch upstream ${BRANCH} git merge ${BRANCH} --no-edit git diff --numstat ${BRANCH} -- | awk "{print \$NF}" else echo "Not in a pull_request event. Skipping PR-specific operations." fi git log --pretty=oneline -10 ' fi - name: Skip For Bug run: | if [[ "$RUN_DOWNSTREAM" == "false" ]]; then echo "Not in a pull_request or test_build event. Skipping..." else docker exec -t $container_name /bin/bash -c ' cd /workspace/PaddleNLP git revert f2477c07272d04244cd3287d1f21c70482a4a85f --no-edit # 套件PR#10413引入bug-待修复 git revert d74c950e15a35b7a100d1688c89318195cc83bca --no-edit # 框架升级后配合修改，当前因框架bug固定commit适配 ' fi - name: Test run: | if [[ "$RUN_DOWNSTREAM" == "false" ]]; then echo "Not in a pull_request or test_build event. Skipping..." else docker exec -t $container_name /bin/bash -c ' ldconfig unlink /usr/bin/python3 ln -sf $(which python${python_version}) /usr/bin/python3 pip config set global.cache-dir "/home/.cache/pip" set -e source $work_dir/../../../proxy cd /workspace/PaddleNLP && git config --global --add safe.directory $PWD echo "Paddle PR#73283 PR#74484 import bug, set paddle_commit=8ae742 to skip " export paddle_whl=https://paddle-qa.bj.bcebos.com/paddle-pipeline/Develop-GpuSome-LinuxCentos-Gcc82-Cuda118-Cudnn86-Trt85-Py310-CINN-Compile/8ae7423e99b2ea96e410968a0ebb3f1795e37205/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl timeout 2h bash scripts/regression/run_ci.sh python${python_version} ${paddle_whl} ' fi - name: Upload Allure-reports & Logs if: always() env: home_path: ${{ github.workspace }}/../../.. bos_file: ${{ github.workspace }}/../../../bos/BosClient.py allure_file: ${{ github.workspace }}/../../../allure-2.19.0/bin/allure run: | if [[ "$RUN_DOWNSTREAM" == "false" ]]; then echo "Not in a pull_request or test_build event. Skipping..." else docker exec -t $container_name /bin/bash -c ' unset http_proxy && unset https_proxy if [ ! -f "${{ env.bos_file }}" ]; then wget -q --no-proxy -O ${{ env.home_path }}/bos_new.tar.gz https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz --no-check-certificate mkdir ${{ env.home_path }}/bos tar xf ${{ env.home_path }}/bos_new.tar.gz -C ${{ env.home_path }}/bos fi if [ ! -f "${{ env.allure_file }}" ]; then wget -q --no-proxy -O ${{ env.home_path }}/allure-2.19.0.zip https://xly-devops.bj.bcebos.com/tools/allure-2.19.0.zip --no-check-certificate unzip -q ${{ env.home_path }}/allure-2.19.0.zip -d ${{ env.home_path }}/ fi if [[ "${{ env.RUN_DOWNSTREAM }}" == "" && -n "${PR_ID}" ]]; then bos_prefix="${PR_ID}/${COMMIT_ID}" elif [[ "${{ env.RUN_DOWNSTREAM }}" == "true" && -n "${PR_ID}" ]]; then bos_prefix="${PR_ID}/${COMMIT_ID}/test_build" else bos_prefix="schedule/$(date +%Y%m%d)" fi cd /workspace/PaddleNLP/model_logs for FILE in /workspace/PaddleNLP/model_logs/*; do file=$(basename "$FILE") python ${{ env.bos_file }} $file paddle-github-action/PR/PaddleNLP/llm/${bos_prefix}/logs echo "$file: https://paddle-github-action.bj.bcebos.com/PR/PaddleNLP/llm/${bos_prefix}/logs/$file" done cd /workspace/PaddleNLP/ ${{ env.allure_file }} generate result -o report tar -czf products.tar.gz report model_logs python ${{ env.bos_file }} products.tar.gz paddle-github-action/PR/PaddleNLP/llm/${bos_prefix}/logs echo "products: https://paddle-github-action.bj.bcebos.com/PR/PaddleNLP/llm/${bos_prefix}/logs/products.tar.gz" ' fi - name: Terminate And Delete the Container if: always() run: | docker rm -f ${{ env.container_name }} 2>/dev/null || true