name: Coverage

on:
  pull_request:
    types: [opened, synchronize]
    branches: [develop, release/**]

permissions: read-all

concurrency:
  group: ${{ github.event.pull_request.number }}-${{ github.workflow }}
  cancel-in-progress: true

env:
  PR_ID: ${{ github.event.pull_request.number }}
  COMMIT_ID: ${{ github.event.pull_request.head.sha }}
  TASK: paddle-CI-${{ github.event.pull_request.number }}-coverage
  ci_scripts: /paddle/ci
  BRANCH: ${{ github.base_ref }}
  work_dir: /paddle
  PADDLE_ROOT: /paddle
  GIT_PR_ID: ${{ github.event.pull_request.number }}
  CI_name: coverage
  CFS_DIR: /home/data/cfs
  no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"

defaults:
  run:
    shell: bash

jobs:
  check-skill:
    name: Check file paths
    if: ${{ github.repository_owner == 'PaddlePaddle' }}
    runs-on:
      group: APPROVAL
    continue-on-error: true
    permissions:
      contents: read
      actions: read
    outputs:
      can-skip: ${{ steps.skip-skill.outputs.should-skip || 'false' }}
    steps:
      - name: Cleanup
        run: |
          rm -rf * .[^.]*
      - name: Checkout
        uses: actions/checkout@v6
      - name: Run skip check
        id: skip-skill
        uses: ./.github/actions/skip-skill
        with:
          ignore-paths: '.agents/skills/**'

  clone:
    name: Coverage clone
    needs: check-skill
    uses: ./.github/workflows/_Clone-linux.yml
    with:
      workflow-name: "coverage"
      clone_dir: Paddle-coverage
      skill-can-skip: ${{ needs.check-skill.outputs.can-skip }}

  build-docker:
    name: Coverage build docker
    needs: clone
    uses: ./.github/workflows/docker.yml
    with:
      clone_dir: Paddle-coverage
      task: coverage

  check-bypass:
    name: Check bypass
    needs: [clone]
    if: needs.clone.outputs.can-skip != 'true'
    uses: ./.github/workflows/check-bypass.yml
    with:
      workflow-name: coverage
    secrets:
      github-token: ${{ secrets.GITHUB_TOKEN }}

  build:
    name: Coverage build
    needs: [clone, build-docker, check-bypass]
    if: ${{ needs.clone.outputs.can-skip != 'true' && needs.check-bypass.outputs.can-skip != 'true' }}
    runs-on:
      group: GZ_BD-CPU
    timeout-minutes: 120
    outputs:
      can-skip: ${{ needs.check-bypass.outputs.can-skip }}

    steps:
      - name: Check docker image and run container
        env:
          CACHE_DIR: "/root/.cache/coverage"
          CCACHE_DIR: "/home/data/shared/.ccache/l1" # L1 cache on machine shared dir
          CCACHE_SECONDARY_STORAGE: "file:///home/data/cfs/.ccache/l2" # L2 cache on cfs
          FLAGS_fraction_of_gpu_memory_to_use: 0.15
          CTEST_PARALLEL_LEVEL: 2
          WITH_GPU: "ON"
          CUDA_ARCH_NAME: Volta
          WITH_AVX: "ON"
          WITH_COVERAGE: "ON"
          COVERALLS_UPLOAD: "ON"
          PADDLE_VERSION: 0.0.0
          CUDA_VISIBLE_DEVICES: 0,1
          WITH_DISTRIBUTE: "ON"
          WITH_PIP_CUDA_LIBRARIES: "OFF"
          WITH_FLAGCX: "ON"
          LITE_GIT_TAG: develop
          WITH_UNITY_BUILD: "ON"
          PY_VERSION: "3.12"
          WITH_SHARED_PHI: "ON"
          WITH_CINN: "ON"
          INFERENCE_DEMO_INSTALL_DIR: /root/.cache/coverage
          CCACHE_MAXSIZE: 50G
          CCACHE_LIMIT_MULTIPLE: 0.8
          ON_INFER: "ON"
          PADDLE_CUDA_INSTALL_REQUIREMENTS: "ON"
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          UT_RUN_TYPE_SETTING: WITHOUT_HYBRID
        run: |
          container_name=${TASK}-build-$(date +%Y%m%d-%H%M%S)
          echo "container_name=${container_name}" >> ${{ github.env }}
          docker_image=${{ needs.build-docker.outputs.docker_coverage_image }}
          docker run -d -t --name ${container_name} \
            -v "/home/data/cfs:/home/data/cfs" \
            -v "/home/data/cfs/.cache:/root/.cache" \
            -v "/home/data/shared:/home/data/shared" \
            -v "/dev/shm:/dev/shm" \
            -v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
            -v ${{ github.workspace }}:/paddle \
            -e CI_name \
            -e BRANCH \
            -e PR_ID \
            -e COMMIT_ID \
            -e work_dir \
            -e PADDLE_ROOT \
            -e GIT_PR_ID \
            -e CACHE_DIR \
            -e CCACHE_DIR \
            -e CCACHE_SECONDARY_STORAGE \
            -e ci_scripts \
            -e FLAGS_fraction_of_gpu_memory_to_use \
            -e CTEST_PARALLEL_LEVEL \
            -e WITH_GPU \
            -e CUDA_ARCH_NAME \
            -e WITH_AVX \
            -e WITH_COVERAGE \
            -e COVERALLS_UPLOAD \
            -e PADDLE_VERSION \
            -e WITH_DISTRIBUTE \
            -e WITH_PIP_CUDA_LIBRARIES \
            -e WITH_FLAGCX \
            -e LITE_GIT_TAG \
            -e WITH_UNITY_BUILD \
            -e PY_VERSION \
            -e WITH_SHARED_PHI \
            -e WITH_CINN \
            -e INFERENCE_DEMO_INSTALL_DIR \
            -e CCACHE_MAXSIZE \
            -e CCACHE_LIMIT_MULTIPLE \
            -e ON_INFER \
            -e PADDLE_CUDA_INSTALL_REQUIREMENTS \
            -e GITHUB_TOKEN \
            -e GITHUB_API_TOKEN \
            -e UT_RUN_TYPE_SETTING \
            -e CFS_DIR \
            -e no_proxy \
            -w /paddle --network host ${docker_image}

      - name: Download paddle.tar.gz and update test branch
        timeout-minutes: 30
        run: |
          docker exec -t ${{ env.container_name }}  /bin/bash -c '
          rm -rf * .[^.]*
          set -e
          echo "Downloading Paddle.tar.gz"
          wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/Paddle-coverage/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
          echo "Extracting Paddle.tar.gz"
          tar -xf Paddle.tar.gz --strip-components=1
          rm Paddle.tar.gz
          git remote -v
          set +e
          git remote add upstream https://github.com/PaddlePaddle/Paddle.git
          set -e
          git config pull.rebase false
          git checkout test
          echo "Pull upstream $BRANCH"
          source ${{ github.workspace }}/../../../proxy
          bash ci/git_pull.sh $BRANCH
          git submodule update
          '

      - name: Build
        run: |
          docker exec -t ${{ env.container_name }}  /bin/bash -c '
          mkdir -p ${CFS_DIR}/.cache/coverage
          mkdir -p ${CFS_DIR}/.ccache/coverage
          source ${{ github.workspace }}/../../../proxy
          bash ${ci_scripts}/cmake-predownload.sh
          bash $ci_scripts/coverage_build.sh bdist_wheel
          '

      - name: Check added unittests
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -c '
          source ~/.bashrc
          source ${{ github.workspace }}/../../../proxy
          bash $ci_scripts/check_added_ut.sh
          '

      - name: Check coverage build size requires approval
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -c '
          source ~/.bashrc
          source ${{ github.workspace }}/../../../proxy
          bash $ci_scripts/coverage_build_size_approval.sh
          '

      - name: Clean up env
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -c '
          source ~/.bashrc
          source ${ci_scripts}/utils.sh; clean_build_files
          Build_Size=$(du -h --max-depth=0 ${work_dir}/build |awk '"'"'{print $1}'"'"')
          echo "Build_Size=${Build_Size}" > ${work_dir}/dist/coverage_build_size
          find ./ -type f -size +200M | xargs du -lh
          rm -rf $(find . -name "*.a")
          rm -rf $(find . -name "*.o")
          rm -rf paddle_inference_install_dir
          rm -rf paddle_inference_c_install_dir
          rm -rf lib.linux-x86_64-${PY_VERSION}
          find ./ -name "eager_generator" -or -name  "kernel_signature_generator" -or -name "eager_legacy_op_function_generator" | xargs rm -rf
          rm -rf ./python/build/lib.linux-x86_64-${PY_VERSION}/
          cd "${work_dir}/build/third_party" && find $(ls | grep -v "dlpack" | grep -v "install" | grep -v "eigen3" | grep -v "gflags") -type f ! -name "*.so" -a ! -name "libdnnl.so*" -delete
          cd /
          tar --use-compress-program="pzstd -1" -cf Paddle.tar.gz paddle
          '

      - name: Upload coverage product
        env:
          home_path: ${{ github.workspace }}/..
          bos_file: ${{ github.workspace }}/../bos_retry/BosClient.py
          paddle_whl: paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -c '
          echo "::group::Install bce-python-sdk"
          source ${{ github.workspace }}/../../../proxy
          python -m pip install bce-python-sdk==0.8.74
          echo "::endgroup::"
          export AK=paddle
          export SK=paddle
          if [ ! -f "${{ env.bos_file }}" ]; then
            wget -q --no-proxy -O ${{ env.home_path }}/bos_retry.tar.gz https://xly-devops.bj.bcebos.com/home/bos_retry.tar.gz --no-check-certificate
            mkdir ${{ env.home_path }}/bos_retry
            tar xf ${{ env.home_path }}/bos_retry.tar.gz -C ${{ env.home_path }}/bos_retry
          fi
          cd /paddle/dist
          mkdir -p ${CFS_DIR}/coverage_bos/${PR_ID}/${COMMIT_ID}
          echo "Uploading coverage build size"
          python ${{ env.bos_file }} coverage_build_size paddle-github-action/PR/coverage/${{ env.PR_ID }}/${{ env.COMMIT_ID }}
          echo "Uploading coverage wheel"
          python ${{ env.bos_file }} ${{ env.paddle_whl }} paddle-github-action/PR/coverage/${{ env.PR_ID }}/${{ env.COMMIT_ID }}
          cd /
          echo "Uploading Paddle.tar.gz"
          cp Paddle.tar.gz ${CFS_DIR}/coverage_bos/${PR_ID}/${COMMIT_ID}
          rm Paddle.tar.gz
          '

      - name: Terminate and delete the container
        if: always()
        run: |
          set +e
          docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
          docker stop ${{ env.container_name }}
          docker rm ${{ env.container_name }}

  test:
    name: Coverage test
    needs: [build, build-docker]
    if: needs.build.outputs.can-skip != 'true'
    runs-on:
      group: BD_BJ-V100
    timeout-minutes: 240
    steps:
      - name: Check docker image and run container
        env:
          CACHE_DIR: "/root/.cache/coverage"
          CCACHE_DIR: "/root/.ccache/coverage"
          FLAGS_fraction_of_gpu_memory_to_use: 0.15
          CTEST_PARALLEL_LEVEL: 2
          WITH_GPU: "ON"
          CUDA_ARCH_NAME: Auto
          WITH_AVX: "ON"
          WITH_COVERAGE: "ON"
          COVERALLS_UPLOAD: "ON"
          PADDLE_VERSION: 0.0.0
          WITH_DISTRIBUTE: "ON"
          WITH_UNITY_BUILD: "ON"
          PY_VERSION: "3.12"
          WITH_SHARED_PHI: "ON"
          WITH_CINN: "ON"
          INFERENCE_DEMO_INSTALL_DIR: /root/.cache/coverage
          CCACHE_MAXSIZE: 200G
          CCACHE_LIMIT_MULTIPLE: 0.8
          FLAGS_PIR_OPTEST: "TRUE"
          ON_INFER: "ON"
          COVERAGE_FILE: ${{ github.workspace }}/build/python-coverage.data
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
          echo "container_name=${container_name}" >> ${{ github.env }}
          docker_image=${{ needs.build-docker.outputs.docker_coverage_image }}
          docker run -d -t --gpus all --name ${container_name} \
            -v "/home/data/cfs:/home/data/cfs" \
            -v "/home/data/cfs/.cache:/root/.cache" \
            -v "/home/data/cfs/.ccache:/root/.ccache" \
            -v "/dev/shm:/dev/shm" \
            -v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
            -v ${{ github.workspace }}:/paddle \
            -e CI_name \
            -e BRANCH \
            -e PR_ID \
            -e COMMIT_ID \
            -e work_dir \
            -e PADDLE_ROOT \
            -e GIT_PR_ID \
            -e CACHE_DIR \
            -e CCACHE_DIR \
            -e ci_scripts \
            -e FLAGS_fraction_of_gpu_memory_to_use \
            -e CTEST_PARALLEL_LEVEL \
            -e WITH_GPU \
            -e CUDA_ARCH_NAME \
            -e WITH_AVX \
            -e WITH_COVERAGE \
            -e COVERALLS_UPLOAD \
            -e PADDLE_VERSION \
            -e WITH_DISTRIBUTE \
            -e WITH_UNITY_BUILD \
            -e PY_VERSION \
            -e WITH_SHARED_PHI \
            -e WITH_CINN \
            -e INFERENCE_DEMO_INSTALL_DIR \
            -e CCACHE_MAXSIZE \
            -e CCACHE_LIMIT_MULTIPLE \
            -e FLAGS_PIR_OPTEST \
            -e ON_INFER \
            -e COVERAGE_FILE \
            -e GITHUB_TOKEN \
            -e GITHUB_API_TOKEN \
            -e CFS_DIR \
            -e no_proxy \
            -w /paddle --network host ${docker_image}

      - name: Download paddle.tar.gz and update test branch
        timeout-minutes: 30
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -c '
          rm -rf * .[^.]*
          set -e
          echo "Downloading Paddle.tar.gz from cfs"
          cp ${CFS_DIR}/coverage_bos/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz .
          echo "Extracting Paddle.tar.gz"
          tar --use-compress-program="pzstd -1" -xf Paddle.tar.gz --strip-components=1
          rm Paddle.tar.gz
          '

      - name: Test
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -c '
          source ${{ github.workspace }}/../../../proxy
          bash $ci_scripts/coverage_test.sh
          TEST_EXIT_CODE=$?
          echo "TEST_EXIT_CODE=${TEST_EXIT_CODE}" >> ${{ github.env }}
          if [[ "$TEST_EXIT_CODE" -ne 0 && "$TEST_EXIT_CODE" -ne 9 ]]; then
              exit $TEST_EXIT_CODE
          fi
          '

      - name: Generate coverage information
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -c '
          source ~/.bashrc
          unset GREP_OPTIONS
          source ${{ github.workspace }}/../../../proxy
          source ${ci_scripts}/utils.sh; check_coverage
          mkdir -p /home/data/cfs/coverage/${PR_ID}/${COMMIT_ID}
          cp build/coverage_files/* /home/data/cfs/coverage/${PR_ID}/${COMMIT_ID}
          '

      - name: Upload coverage report
        uses: codecov/codecov-action@v6
        with:
          directory: build/coverage_files
          verbose: true

      - name: Determine whether the coverage rate reaches 90%
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -c '
          source ~/.bashrc
          unset GREP_OPTIONS
          source ${{ github.workspace }}/../../../proxy
          source ${ci_scripts}/utils.sh
          bash $ci_scripts/coverage_judge.sh
          COVERAGE_EXIT_CODE=$?
          echo $COVERAGE_EXIT_CODE
          if [ "$COVERAGE_EXIT_CODE" -ne 0 ]; then
            echo "Coverage check failed, unit tests have all passed, please do not rerun, check whether the newly added code lines are fully covered by unit tests. If you have any questions, please contact XieYunshen."
            exit $COVERAGE_EXIT_CODE
          else
            echo "Coverage passed"
          fi
          '

      - name: Terminate and delete the container
        if: always()
        run: |
          set +e
          rm Paddle.tar.gz
          docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
          docker stop ${{ env.container_name }}
          docker rm ${{ env.container_name }}