name: Linux-DCU

on:
  workflow_call:
    inputs:
      docker_dcu_image:
        type: string
        required: true
      clone-can-skip:
        type: string
        required: false
        default: "false"

env:
  docker_image: ${{ inputs.docker_dcu_image }}
  PR_ID: ${{ github.event.pull_request.number }}
  COMMIT_ID: ${{ github.event.pull_request.head.sha }}
  ci_scripts: /paddle/ci
  ci_scripts_runner: ${{ github.workspace }}/ci
  work_dir: /paddle
  PADDLE_ROOT: /paddle
  BRANCH: ${{ github.event.pull_request.base.ref }}
  PADDLE_BRANCH: ${{ github.event.pull_request.base.ref }}
  CI_name: dcu
  no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"

defaults:
  run:
    shell: bash

jobs:
  check-bypass:
    name: Check bypass
    uses: ./.github/workflows/check-bypass.yml
    with:
      workflow-name: "dcu"
    secrets:
      github-token: ${{ secrets.GITHUB_TOKEN }}

  build:
    name: Build
    needs: [check-bypass]
    if: ${{ needs.check-bypass.outputs.can-skip != 'true' && inputs.clone-can-skip != 'true' }}
    env:
      TASK: paddle-CI-${{ github.event.pull_request.number }}-dcu_build
    runs-on:
      group: HK-CPU
    timeout-minutes: 180
    steps:
      - name: Check docker image and run container
        env:
          WITH_DOC: "OFF"
          WITH_GPU: "OFF"
          WITH_ROCM: "ON"
          WITH_RCCL: "ON"
          WITH_XPU: "OFF"
          WITH_TENSORRT: "OFF"
          WITH_COVERAGE: "OFF"
          COVERALLS_UPLOAD: "OFF"
          CMAKE_BUILD_TYPE: "Release"
          WITH_AVX: "ON"
          WITH_MKL: "ON"
          WITH_ARM: "OFF"
          WITH_CACHE: "ON"
          RUN_TEST: "ON"
          WITH_TESTING: "ON"
          CI_SKIP_CPP_TEST: "OFF"
          WITH_DISTRIBUTE: "ON"
          CMAKE_EXPORT_COMPILE_COMMANDS: "ON"
          GIT_PR_ID: ${{ github.event.pull_request.number }}
          PADDLE_VERSION: 0.0.0
          PY_VERSION: "3.10"
          CACHE_DIR: /root/.cache
          CCACHE_DIR: /root/.ccache
          CCACHE_MAXSIZE: 150G
          CCACHE_LIMIT_MULTIPLE: 0.8
          GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          home_dir: ${{ github.workspace }}/../../../..
        run: |
          container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
          echo "container_name=${container_name}" >> ${{ github.env }}
          docker run --shm-size=128G \
            --ulimit nofile=102400:102400 -d -t --name ${container_name} \
            -v $home_dir/.cache:/root/.cache \
            -v $home_dir/.ccache:/root/.ccache \
            -v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
            -v ${{ github.workspace }}:/paddle \
            -e PR_ID \
            -e COMMIT_ID \
            -e work_dir \
            -e PADDLE_ROOT \
            -e WITH_DOC \
            -e WITH_GPU \
            -e WITH_ROCM \
            -e WITH_RCCL \
            -e WITH_XPU \
            -e WITH_TENSORRT \
            -e WITH_COVERAGE \
            -e COVERALLS_UPLOAD \
            -e CMAKE_BUILD_TYPE \
            -e WITH_MKL \
            -e WITH_AVX \
            -e WITH_ARM  \
            -e RUN_TEST \
            -e WITH_TESTING \
            -e CI_SKIP_CPP_TEST \
            -e WITH_DISTRIBUTE \
            -e CACHE_DIR \
            -e CCACHE_DIR \
            -e CCACHE_MAXSIZE \
            -e CCACHE_LIMIT_MULTIPLE \
            -e ci_scripts \
            -e BRANCH \
            -e PADDLE_BRANCH \
            -e PADDLE_VERSION \
            -e no_proxy \
            -e CMAKE_EXPORT_COMPILE_COMMANDS \
            -e PY_VERSION \
            -e GIT_PR_ID \
            -e GITHUB_API_TOKEN \
            -e CI_name \
            -w /paddle --network host ${docker_image} /bin/bash

      - name: Download paddle.tar.gz and update test branch
        timeout-minutes: 30
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -c '
          ulimit -n 102400
          rm -rf * .[^.]*
          set -e
          echo "Downloading Paddle.tar.gz"
          wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/Paddle/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
          echo "Extracting Paddle.tar.gz"
          git config --global --add safe.directory ${work_dir}
          tar -xf Paddle.tar.gz  --strip-components=1
          git submodule foreach "git config --global --add safe.directory \$toplevel/\$sm_path"
          rm Paddle.tar.gz
          git config --global user.name "PaddleCI"
          git config --global user.email "paddle_ci@example.com"
          git remote add upstream https://github.com/PaddlePaddle/Paddle.git
          git config pull.rebase false
          git checkout test
          echo "Pull upstream develop or target branch"
          git pull upstream $BRANCH --no-edit
          git submodule update
          '

      - name: Run build
        env:
          work_dir: ${{ github.workspace }}
          PADDLE_ROOT: ${{ github.workspace }}
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -c '
          source /opt/dtk-24.04.1/env.sh
          bash ${ci_scripts}/run_setup.sh bdist_wheel
          EXCODE=$?
          python -m pip install bce-python-sdk==0.8.74
          rm -rf `find . -name "*.a"`
          rm -rf `find . -name "*.o"`
          exit $EXCODE
          '

      - name: Upload build.tar.gz and paddle_whl to bos
        env:
          AK: paddle
          SK: paddle
          home_path: ${{ github.workspace }}/..
          bos_file: ${{ github.workspace }}/../bos/BosClient.py
          paddle_whl: paddlepaddle_dcu-0.0.0-cp310-cp310-linux_x86_64.whl
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -c '
          if [ ! -f "${{ env.bos_file }}" ]; then
            wget -q --no-proxy -O ${{ env.home_path }}/bos_new.tar.gz https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz --no-check-certificate
            mkdir ${{ env.home_path }}/bos
            tar xf ${{ env.home_path }}/bos_new.tar.gz -C ${{ env.home_path }}/bos
          fi
          apt install zstd -y
          cd ..
          tar --use-compress-program="pzstd -5" -cf build.tar.gz paddle
          echo "Uploading build.tar.gz"
          python ${{ env.bos_file }} build.tar.gz paddle-github-action/PR/dcu/${{ env.PR_ID }}/${{ env.COMMIT_ID }}
          echo "Uploading paddle_whl"
          cp /paddle/dist/${{ env.paddle_whl }} .
          python ${{ env.bos_file }} ${{ env.paddle_whl }} paddle-github-action/PR/dcu/${{ env.PR_ID }}/${{ env.COMMIT_ID }}
          '

      - name: Terminate and delete the container
        if: always()
        run: |
          docker stop ${container_name}
          docker rm ${container_name}

  test:
    name: Test
    needs: build
    env:
      TASK: paddle-CI-${{ github.event.pull_request.number }}-dcu_test
    runs-on:
      group: dcu-z100
    timeout-minutes: 120

    steps:
      - name: Determine the runner
        run: |
          set -x
          runner_name=`(echo $PWD|awk -F '/' '{print $3}')`
          wget -q https://xly-devops.bj.bcebos.com/utils.sh
          source utils.sh
          determine_dcu_runner ${runner_name}

      - name: Check docker image and run container
        env:
          CMAKE_BUILD_TYPE: Release
          WITH_ROCM: "ON"
          WITH_RCCL: "ON"
          WITH_AVX: "ON"
          WITH_MKL: "ON"
          IF_DCU: "OFF"
          WITH_TENSORRT: "OFF"
          WITH_XPU: "OFF"
          WITH_CINN: "ON"
          GIT_PR_ID: ${{ github.event.pull_request.number }}
          PADDLE_VERSION: 0.0.0
          WITH_TESTING: "ON"
          WITH_DISTRIBUTE: "ON"
          HIP_VISIBLE_DEVICES: ${{ env.HIP_VISIBLE_DEVICES }}
          PY_VERSION: "3.10"
          CACHE_DIR: /root/.cache
          CCACHE_DIR: /root/.ccache
          CCACHE_MAXSIZE: 150G
          CCACHE_LIMIT_MULTIPLE: 0.8
          GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          home_dir: ${{ github.workspace }}/../../../..
        run: |
          container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
          echo "container_name=${container_name}" >> ${{ github.env }}
          docker run --ulimit nofile=102400 --ulimit core=-1 --shm-size=32g -d -t --name ${container_name} \
            -v /ssd1/cibuild/.cache:/root/.cache \
            -v /ssd1/cibuild/.ccache:/root/.ccache \
            -v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
            -v ${{ github.workspace }}:/paddle \
            --device=/dev/kfd \
            --device=/dev/dri \
            --device=/dev/mkfd \
            --group-add video \
            --shm-size=32g \
            -e BRANCH \
            -e PR_ID \
            -e COMMIT_ID \
            -e work_dir \
            -e PADDLE_ROOT \
            -e WITH_ROCM \
            -e WITH_RCCL \
            -e WITH_AVX \
            -e WITH_MKL \
            -e WITH_TENSORRT \
            -e WITH_XPU \
            -e IF_DCU \
            -e HIP_VISIBLE_DEVICES \
            -e WITH_RCCL \
            -e CMAKE_BUILD_TYPE \
            -e GIT_PR_ID \
            -e PADDLE_VERSION \
            -e WITH_TESTING \
            -e WITH_DISTRIBUTE \
            -e PY_VERSION \
            -e CACHE_DIR \
            -e CCACHE_DIR \
            -e WITH_INFERENCE_API_TEST \
            -e CCACHE_MAXSIZE \
            -e CCACHE_LIMIT_MULTIPLE \
            -e ci_scripts \
            -e WITH_AVX \
            -e no_proxy \
            -e GITHUB_API_TOKEN \
            -e CI_name \
            -w /paddle --network host ${docker_image} /bin/bash

      - name: Download build.tar.gz
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -c '
          rm -rf ./*
          wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/dcu/${PR_ID}/${COMMIT_ID}/build.tar.gz --no-check-certificate
          tar --use-compress-program="pzstd" -xf build.tar.gz --strip-components=1
          rm build.tar.gz
          '

      - name: Run test
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -c '
          export PATH=/usr/local/bin:${PATH}
          ln -sf $(which python3.10) /usr/local/bin/python
          ln -sf $(which pip3.10) /usr/local/bin/pip
          pip3.10 install ./dist/paddlepaddle_dcu-0.0.0-cp310-cp310-linux_x86_64.whl
          wget -q --no-proxy https://paddle-device.bj.bcebos.com/dcu/hyhal-Z100.tar.gz
          tar -zxf hyhal-Z100.tar.gz -C /opt
          source /opt/dtk-24.04.1/env.sh
          bash -x ${ci_scripts}/dcu_test.sh
          '

      - name: Terminate and delete the container
        if: always()
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -c '
          rm -rf * .[^.]*
          '
          docker stop ${container_name}
          docker rm ${container_name}