name: PR-CI-Inference

on:
  workflow_call:
    inputs:
      docker_inference_image:
        type: string
        required: true
      clone-can-skip:
        type: string
        required: false
        default: "false"

env:
  PR_ID: ${{ github.event.pull_request.number }}
  COMMIT_ID: ${{ github.event.pull_request.head.sha }}
  ci_scripts: /paddle/ci
  work_dir: /paddle
  PADDLE_ROOT: /paddle
  BRANCH: ${{ github.event.pull_request.base.ref }}
  CFS_DIR: /home/data/cfs
  no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
  docker_image: ${{ inputs.docker_inference_image }}

defaults:
  run:
    shell: bash

jobs:
  check-bypass:
    name: Check bypass
    uses: ./.github/workflows/check-bypass.yml
    with:
      workflow-name: "inference"
    secrets:
      github-token: ${{ secrets.GITHUB_TOKEN }}

  build:
    name: Build
    needs: [check-bypass]
    if: ${{ needs.check-bypass.outputs.can-skip != 'true' && inputs.clone-can-skip != 'true' }}
    env:
      TASK: paddle-CI-${{ github.event.pull_request.number }}-inference_build
    runs-on:
      group: GZ_BD-CPU
    timeout-minutes: 120

    steps:
      - name: Check docker image and run container
        env:
          FLAGS_fraction_of_gpu_memory_to_use: "0.15"
          CTEST_OUTPUT_ON_FAILURE: 1
          CTEST_PARALLEL_LEVEL: 4
          WITH_GPU: "ON"
          WITH_AVX: "ON"
          WITH_TESTING: "OFF"
          COVERALLS_UPLOAD: "ON"
          PADDLE_VERSION: 0.0.0
          CUDA_VISIBLE_DEVICES: "0,1"
          GIT_PR_ID: ${{ github.event.pull_request.number }}
          INFERENCE_DEMO_INSTALL_DIR: /root/.cache/inference_demo
          WITH_PYTHON: "OFF"
          PY_VERSION: "3.10"
          WITH_ONNXRUNTIME: "ON"
          WITH_UNITY_BUILD: "ON"
          WITH_NVCC_LAZY: "OFF"
          PYTHON_ABI: cp310-cp310
          CACHE_DIR: /root/.cache/inference
          CCACHE_DIR: "/home/data/shared/.ccache/l1" # L1 cache on machine shared dir
          CCACHE_SECONDARY_STORAGE: "file:///home/data/cfs/.ccache/l2" # L2 cache on cfs
          CCACHE_MAXSIZE: 50G
          CCACHE_LIMIT_MULTIPLE: "0.8"
          CUDA_ARCH_NAME: Pascal
          PADDLE_CUDA_INSTALL_REQUIREMENTS: "ON"
        run: |
          container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
          echo  "container_name=${container_name}" >> ${{ github.env }}
          docker run -d -t --name ${container_name} \
          -v "/home/data/cfs:/home/data/cfs" \
          -v "/home/data/cfs/.cache:/root/.cache" \
          -v "/home/data/cfs/.ccache:/root/.ccache" \
          -v "/home/data/cfs/.ccache/inference:/root/.ccache/inference" \
          -v "/dev/shm:/dev/shm" \
          -v "${{ github.workspace }}/../../..:/action-runner" \
          -e BRANCH \
          -e PR_ID \
          -e COMMIT_ID \
          -e work_dir \
          -e PADDLE_ROOT \
          -e FLAGS_fraction_of_gpu_memory_to_use \
          -e CTEST_OUTPUT_ON_FAILURE \
          -e CTEST_PARALLEL_LEVEL \
          -e WITH_GPU \
          -e WITH_AVX \
          -e CUDA_VISIBLE_DEVICES \
          -e INFERENCE_DEMO_INSTALL_DIR \
          -e WITH_PYTHON \
          -e WITH_ONNXRUNTIME \
          -e WITH_NVCC_LAZY \
          -e PYTHON_ABI \
          -e COVERALLS_UPLOAD \
          -e GIT_PR_ID \
          -e PADDLE_VERSION \
          -e WITH_TESTING \
          -e PY_VERSION \
          -e WITH_UNITY_BUILD \
          -e CACHE_DIR \
          -e CCACHE_DIR \
          -e CCACHE_MAXSIZE \
          -e CCACHE_LIMIT_MULTIPLE \
          -e ci_scripts \
          -e CUDA_ARCH_NAME \
          -e PADDLE_CUDA_INSTALL_REQUIREMENTS \
          -e CFS_DIR \
          -e no_proxy \
          -w /paddle --network host ${docker_image} /bin/bash

      - name: Download paddle.tar.gz and update test branch
        timeout-minutes: 30
        run: |
          docker exec -t ${container_name} /bin/bash -c '
          rm -rf * .[^.]*
          set -e
          echo "Downloading Paddle.tar.gz"
          wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/Paddle-build/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
          echo "Extracting Paddle.tar.gz"
          tar -xf Paddle.tar.gz --strip-components=1
          rm Paddle.tar.gz
          git config --global user.name "PaddleCI"
          git config --global user.email "paddle_ci@example.com"
          git config --global --add safe.directory "*"
          git remote -v
          set +e
          git remote add upstream https://github.com/PaddlePaddle/Paddle.git
          set -e
          source /action-runner/proxy
          git config pull.rebase false
          git checkout test
          echo "Pull upstream develop"
          bash ci/git_pull.sh $BRANCH
          git submodule update
          '

      - name: Run build
        run: |
          docker exec -t ${container_name} /bin/bash -c '
          source /action-runner/proxy
          bash ${ci_scripts}/cmake-predownload.sh
          bash ${ci_scripts}/inference_build.sh
          EXCODE=$?
          exit $EXCODE
          '

      - name: Upload build.tar.gz
        env:
          home_path: /action-runner
          bos_file: /action-runner/bos_retry/BosClient.py
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -c '
          echo "::group::Install bce-python-sdk"
          source /action-runner/proxy
          python -m pip install bce-python-sdk==0.8.74
          echo "::endgroup::"
          rm -rf `find . -name "*.o"`
          export AK=paddle
          export SK=paddle
          if [ ! -f "${{ env.bos_file }}" ]; then
            wget -q --tries=5 --no-proxy -O ${{ env.home_path }}/bos_retry.tar.gz https://xly-devops.bj.bcebos.com/home/bos_retry.tar.gz --no-check-certificate
            mkdir ${{ env.home_path }}/bos_retry
            tar xf ${{ env.home_path }}/bos_retry.tar.gz -C ${{ env.home_path }}/bos_retry
          fi
          source /action-runner/unproxy
          cd /
          tar --use-compress-program="pigz -1" -cpPf build.tar.gz paddle
          echo "Uploading build.tar.gz to cfs"
          mkdir -p ${CFS_DIR}/inference_bos/${PR_ID}/${COMMIT_ID}
          cp build.tar.gz ${CFS_DIR}/inference_bos/${PR_ID}/${COMMIT_ID} && echo success
          rm build.tar.gz
          '

      - name: Terminate and delete the container
        if: always()
        run: |
          # docker exec -t ${container_name} /bin/bash -c 'rm -rf * .[^.]*'
          docker stop ${container_name}
          docker rm ${container_name}

  test:
    name: Test
    needs: build
    env:
      TASK: paddle-CI-${{ github.event.pull_request.number }}-inference_test
    runs-on:
      group: Inference-P4
    timeout-minutes: 120

    steps:
      - name: Check docker image and run container
        env:
          FLAGS_fraction_of_gpu_memory_to_use: "0.15"
          CTEST_OUTPUT_ON_FAILURE: 1
          CTEST_PARALLEL_LEVEL: 4
          WITH_GPU: "ON"
          WITH_AVX: "ON"
          WITH_TESTING: "OFF"
          COVERALLS_UPLOAD: "ON"
          PADDLE_VERSION: 0.0.0
          CUDA_VISIBLE_DEVICES: "0,1"
          GIT_PR_ID: ${{ github.event.pull_request.number }}
          INFERENCE_DEMO_INSTALL_DIR: /root/.cache/inference_demo
          WITH_PYTHON: "OFF"
          PY_VERSION: "3.10"
          WITH_ONNXRUNTIME: "ON"
          WITH_UNITY_BUILD: "ON"
          WITH_NVCC_LAZY: "OFF"
          PYTHON_ABI: cp310-cp310
          CACHE_DIR: /root/.cache/inference
          CCACHE_DIR: /root/.ccache/inference
          CCACHE_MAXSIZE: 150G
          CCACHE_LIMIT_MULTIPLE: "0.8"
          CUDA_ARCH_NAME: Auto
          PADDLE_CUDA_INSTALL_REQUIREMENTS: "ON"
          GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
          echo "container_name=${container_name}" >> ${{ github.env }}
          docker pull $docker_image
          docker run -d -t --gpus all --name ${container_name} \
          -v "/home/data/cfs:/home/data/cfs" \
          -v "/home/data/cfs/.cache:/root/.cache" \
          -v "/home/data/shared:/home/data/shared" \
          -v "/dev/shm:/dev/shm" \
          -v "${{ github.workspace }}/../../..:/action-runner" \
          -e BRANCH \
          -e PR_ID \
          -e COMMIT_ID \
          -e work_dir \
          -e PADDLE_ROOT \
          -e FLAGS_fraction_of_gpu_memory_to_use \
          -e CTEST_OUTPUT_ON_FAILURE \
          -e CTEST_PARALLEL_LEVEL \
          -e WITH_GPU \
          -e WITH_AVX \
          -e CUDA_VISIBLE_DEVICES \
          -e INFERENCE_DEMO_INSTALL_DIR \
          -e WITH_PYTHON \
          -e WITH_ONNXRUNTIME \
          -e WITH_NVCC_LAZY \
          -e PYTHON_ABI \
          -e COVERALLS_UPLOAD \
          -e GIT_PR_ID \
          -e PADDLE_VERSION \
          -e WITH_TESTING \
          -e PY_VERSION \
          -e WITH_UNITY_BUILD \
          -e CACHE_DIR \
          -e CCACHE_DIR \
          -e CCACHE_SECONDARY_STORAGE \
          -e CCACHE_MAXSIZE \
          -e CCACHE_LIMIT_MULTIPLE \
          -e ci_scripts \
          -e CUDA_ARCH_NAME \
          -e PADDLE_CUDA_INSTALL_REQUIREMENTS \
          -e GITHUB_API_TOKEN \
          -e CFS_DIR \
          -e no_proxy \
          -w /paddle --network host ${docker_image} /bin/bash

      - name: Download paddle.tar.gz
        timeout-minutes: 30
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -c '
          set -e
          echo "Downloading build.tar.gz from cfs"
          cp ${CFS_DIR}/inference_bos/${PR_ID}/${COMMIT_ID}/build.tar.gz .
          echo "Extracting build.tar.gz"
          tar --use-compress-program="pigz -1" -xpf build.tar.gz --strip-components=1
          rm build.tar.gz
          export PATH=$(pwd)/run_env:${PATH}
          echo "export PATH=${PATH}" >> ~/.bashrc
          echo "::group::Install dependencies"
          source /action-runner/proxy
          pip install -r "python/unittest_py/requirements.txt"
          echo "::endgroup::"
          '

      - name: Test fluid library for inference
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -c '
          source ~/.bashrc
          source ${ci_scripts}/utils.sh
          init
          source /action-runner/proxy
          test_fluid_lib
          '

      - name: Test go inference api
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -c '
          source ~/.bashrc
          source ${ci_scripts}/utils.sh
          init
          source /action-runner/proxy
          test_go_inference_api
          '

      - name: Check approvals of unittest
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -c '
          source ~/.bashrc
          source ${ci_scripts}/utils.sh
          init
          source /action-runner/proxy
          check_approvals_of_unittest 3
          '

      - name: Terminate and delete the container
        if: always()
        run: |
          # docker exec -t ${container_name} /bin/bash -c 'rm -rf * .[^.]*'
          docker stop ${container_name}
          docker rm ${container_name}