name: Base Test
description: "Run Base Tests"

on:
  workflow_call:
    inputs:
      DOCKER_IMAGE:
        description: "Build Images"
        required: true
        type: string
        default: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:cuda126-py310"
      FASTDEPLOY_ARCHIVE_URL:
        description: "URL of the compressed FastDeploy code archive."
        required: true
        type: string
      FASTDEPLOY_WHEEL_URL:
        description: "URL of the FastDeploy Wheel."
        required: true
        type: string
      CACHE_DIR:
        description: "Cache Dir Use"
        required: false
        type: string
        default: ""
      MODEL_CACHE_DIR:
        description: "Cache Dir Use"
        required: false
        type: string
        default: ""
    secrets:
      github-token:
        required: true

jobs:
  check_bypass:
    uses: ./.github/workflows/check-bypass.yml
    secrets:
      github-token: ${{ secrets.github-token }}
    with:
      workflow-name: base_test

  base_tests:
    runs-on: [self-hosted, GPU-h20-1Cards]
    needs: check_bypass
    if: ${{ inputs.FASTDEPLOY_WHEEL_URL != '' && needs.check_bypass.outputs.can-skip != 'true' }}
    timeout-minutes: 60
    steps:
      - name: Code Prepare
        shell: bash
        env:
          docker_image: ${{ inputs.DOCKER_IMAGE }}
          fd_archive_url: ${{ inputs.FASTDEPLOY_ARCHIVE_URL }}
        run: |
          set -x
          REPO="https://github.com/${{ github.repository }}.git"
          FULL_REPO="${{ github.repository }}"
          REPO_NAME="${FULL_REPO##*/}"
          BASE_BRANCH="${{ github.base_ref }}"
          docker pull ${docker_image}
          # Clean the repository directory before starting
          docker run --rm --net=host -v $(pwd):/workspace -w /workspace \
          -e "REPO_NAME=${REPO_NAME}" \
          ${docker_image} /bin/bash -c '
              CLEAN_RETRIES=3
              CLEAN_COUNT=0

              while [ $CLEAN_COUNT -lt $CLEAN_RETRIES ]; do
                echo "Attempt $((CLEAN_COUNT+1)) to remove ${REPO_NAME}* ..."
                rm -rf "${REPO_NAME}"* || true
                sleep 2

                # Check if anything matching ${REPO_NAME}* still exists
                if ! ls "${REPO_NAME}"* >/dev/null 2>&1; then
                  echo "All ${REPO_NAME}* removed successfully"
                  break
                fi

                CLEAN_COUNT=$((CLEAN_COUNT + 1))
              done

              if ls "${REPO_NAME}"* >/dev/null 2>&1; then
                echo "ERROR: Failed to clean ${REPO_NAME}* after multiple attempts"
                ls -ld "${REPO_NAME}"*
                echo "Attempting force cleanup with find..."
                find /workspace -mindepth 1 -maxdepth 1 -name "${REPO_NAME}*" -type d -exec chmod -R u+rwx {} \; -exec rm -rf {} + 2>/dev/null || true
                if ls "${REPO_NAME}"* >/dev/null 2>&1; then
                  echo "ERROR: Force cleanup still failed"
                  exit 1
                else
                  echo "Force cleanup succeeded"
                fi
              fi
            '

          # Download with retry and validation
          MAX_RETRIES=3
          RETRY_COUNT=0
          while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do
            if wget -q --no-proxy ${fd_archive_url} && [ -f FastDeploy.tar.gz ] && [ -s FastDeploy.tar.gz ]; then
              echo "Download successful, file size: $(stat -c%s FastDeploy.tar.gz) bytes"
              break
            else
              RETRY_COUNT=$((RETRY_COUNT + 1))
              echo "Download failed or file is empty, retry $RETRY_COUNT/$MAX_RETRIES..."
              rm -f FastDeploy.tar.gz
              sleep 2
            fi
          done

          if [ ! -f FastDeploy.tar.gz ] || [ ! -s FastDeploy.tar.gz ]; then
            echo "ERROR: Failed to download FastDeploy.tar.gz after $MAX_RETRIES attempts"
            exit 1
          fi

          # Verify tar.gz integrity before extraction
          if ! tar -tzf FastDeploy.tar.gz > /dev/null 2>&1; then
            echo "ERROR: FastDeploy.tar.gz is corrupted or incomplete"
            exit 1
          fi

          tar --no-same-owner -xf FastDeploy.tar.gz || {
            echo "ERROR: Failed to extract archive"
            exit 1
          }

          rm -rf FastDeploy.tar.gz
          cd FastDeploy
          git config --global user.name "FastDeployCI"
          git config --global user.email "fastdeploy_ci@example.com"
          git log -n 3 --oneline

      - name: Run FastDeploy Base Tests
        shell: bash
        env:
          docker_image: ${{ inputs.DOCKER_IMAGE }}
          fastdeploy_wheel_url: ${{ inputs.FASTDEPLOY_WHEEL_URL }}
          CACHE_DIR: ${{ inputs.CACHE_DIR }}
          MODEL_CACHE_DIR: ${{ inputs.MODEL_CACHE_DIR }}
        run: |
          runner_name="${{ runner.name }}"
          CARD_ID=$(echo "${runner_name}" | awk -F'-' '{print $NF}')
          DEVICES=$(echo "$CARD_ID" | fold -w1 | paste -sd,)
          DEVICE_PORT=$(echo "$DEVICES" | cut -d',' -f1)

          FLASK_PORT=$((8068 + DEVICE_PORT * 100))
          FD_API_PORT=$((8088 + DEVICE_PORT * 100))
          FD_ENGINE_QUEUE_PORT=$((8058 + DEVICE_PORT * 100))
          FD_METRICS_PORT=$((8078 + DEVICE_PORT * 100))
          FD_CACHE_QUEUE_PORT=$((8098 + DEVICE_PORT * 100))
          echo "Test ENV Parameter:"
          echo "========================================================="
          echo "FLASK_PORT=${FLASK_PORT}"
          echo "FD_API_PORT=${FD_API_PORT}"
          echo "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}"
          echo "FD_METRICS_PORT=${FD_METRICS_PORT}"
          echo "FD_CACHE_QUEUE_PORT=${FD_CACHE_QUEUE_PORT}"
          echo "DEVICES=${DEVICES}"
          echo "========================================================="

          CACHE_DIR="${CACHE_DIR:-$(dirname "$(dirname "${{ github.workspace }}")")}"
          echo "CACHE_DIR is set to ${CACHE_DIR}"
          if [ ! -f "${CACHE_DIR}/gitconfig" ]; then
            touch "${CACHE_DIR}/gitconfig"
          fi
          if [ ! -d "${MODEL_CACHE_DIR}" ]; then
            echo "Error: MODEL_CACHE_DIR '${MODEL_CACHE_DIR}' does not exist."
            exit 1
          fi

          PORTS=($FLASK_PORT $FD_API_PORT $FD_ENGINE_QUEUE_PORT $FD_METRICS_PORT $FD_CACHE_QUEUE_PORT)
          LOG_FILE="./port_cleanup_$(date +%Y%m%d_%H%M%S).log"
          echo "==== LOG_FILE is ${LOG_FILE} ===="

          echo "==== PORT CLEAN BEFORE TASK RUN ====" | tee -a $LOG_FILE

          for port in "${PORTS[@]}"; do
              PIDS=$(lsof -t -i :$port || true)
              if [ -n "$PIDS" ]; then
                  echo "Port $port is occupied by PID(s): $PIDS" | tee -a $LOG_FILE
                  echo "$PIDS" | xargs -r kill -9
                  echo "Port $port cleared" | tee -a $LOG_FILE
              else
                  echo "Port $port is free" | tee -a $LOG_FILE
              fi
          done

          echo "==== PORT CLEAN COMPLETE ====" | tee -a $LOG_FILE

          echo "========================================================="
          echo "Ensuring no stale container named ${runner_name} ..."
          if [ "$(docker ps -a -q -f name=${runner_name})" ]; then
            echo "Removing stale container: ${runner_name}"
            docker rm -f ${runner_name} || true
          fi

          docker run --rm --net=host \
          --shm-size=64g \
          --sysctl kernel.msgmax=1048576 \
          --sysctl kernel.msgmnb=268435456 \
          --name ${runner_name} \
          -v $(pwd):/workspace \
          -w /workspace \
          -e fastdeploy_wheel_url=${fastdeploy_wheel_url} \
          -e "FD_API_PORT=${FD_API_PORT}" \
          -e "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}" \
          -e "FD_METRICS_PORT=${FD_METRICS_PORT}" \
          -e "FLASK_PORT=${FLASK_PORT}" \
          -e "FD_CACHE_QUEUE_PORT=${FD_CACHE_QUEUE_PORT}" \
          -v "${MODEL_CACHE_DIR}:/MODELDATA" \
          -v "${CACHE_DIR}/gitconfig:/etc/gitconfig:ro" \
          -v "${CACHE_DIR}/.cache:/root/.cache" \
          -v "${CACHE_DIR}/ConfigDir:/root/.config" \
          -e TZ="Asia/Shanghai" \
          -e "no_proxy=localhost,127.0.0.1,0.0.0.0,bcebos.com,.bcebos.com,bj.bcebos.com,su.bcebos.com,paddle-ci.gz.bcebos.com,apiin.im.baidu.com,baidu-int.com,.baidu.com,aliyun.com,gitee.com,pypi.tuna.tsinghua.edu.cn,.tuna.tsinghua.edu.cn" \
          --gpus '"device='"${DEVICES}"'"' ${docker_image} /bin/bash -xc '
          python -m pip install https://paddle-qa.bj.bcebos.com/paddle-pipeline/Release-TagBuild-Training-Linux-Gpu-Cuda12.6-Cudnn9.5-Trt10.5-Mkl-Avx-Gcc11-SelfBuiltPypiUse/2b9f8b689bc8988f97a5ede056c8c81bfa0332c2/paddlepaddle_gpu-3.3.1.post20260420+2b9f8b689bc-cp310-cp310-linux_x86_64.whl --extra-index-url https://www.paddlepaddle.org.cn/packages/stable/cu126/

          pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple

          python -m pip install ${fastdeploy_wheel_url}
          python -m pip install pytest

          wget --no-proxy https://paddle-qa.bj.bcebos.com/zhengtianyu/tools/llm-deploy-linux-amd64
          chmod +x ./llm-deploy-linux-amd64
          ./llm-deploy-linux-amd64 -python python3.10 \
          -model_name ERNIE-4.5-0.3B-Paddle \
          -model_path /MODELDATA \
          --skip install,model

          git config --global --add safe.directory /workspace/FastDeploy
          cd FastDeploy
          pushd tests/ce/deploy
          ps -ef | grep "${FD_CACHE_QUEUE_PORT}" | grep -v grep | awk "{print \$2}" | xargs -r kill -9
          ps -ef | grep "${FD_ENGINE_QUEUE_PORT}" | grep -v grep | awk "{print \$2}" | xargs -r kill -9
          python3.10 deploy.py > dd.log 2>&1 &
          sleep 3
          curl -X POST http://0.0.0.0:${FLASK_PORT}/start \
            -H "Content-Type: application/json" \
            -d "{\"--model\": \"/MODELDATA/ERNIE-4.5-0.3B-Paddle\"}"

          check_service() {
            local timeout=${1:-90}
            local url="http://localhost:${FLASK_PORT}/wait_for_infer?timeout=${timeout}"
            local resp

            resp=$(curl -s -X POST "$url")

            if echo "$resp" | grep -q "服务启动超时"; then
              exit 8
            fi
          }

          check_service 90
          popd

          pushd tests/ce/server
          export URL=http://localhost:${FD_API_PORT}/v1/chat/completions
          export TEMPLATE=TOKEN_LOGPROB
          TEST_EXIT_CODE=0
          python -m pytest -sv test_base_chat.py test_compare_top_logprobs.py test_logprobs.py test_params_boundary.py test_seed_usage.py test_stream.py test_evil_cases.py test_completions.py test_return_token_ids.py test_update_weight.py || TEST_EXIT_CODE=1
          curl -X POST http://0.0.0.0:${FLASK_PORT}/switch \
            -H "Content-Type: application/json" \
            -d "{\"--model\": \"/MODELDATA/ERNIE-4.5-0.3B-Paddle\", \"--early-stop-config\": \"{\\\"enable_early_stop\\\":true, \\\"window_size\\\":6, \\\"threshold\\\":0.93}\"}"
          check_service 90
          python -m pytest -sv test_repetition_early_stop.py || TEST_EXIT_CODE=1

          curl -X POST http://0.0.0.0:${FLASK_PORT}/switch \
            -H "Content-Type: application/json" \
            -d "{ \"--model\": \"/MODELDATA/ERNIE-4.5-0.3B-Paddle\", \"--max-concurrency\": 5, \"--max-waiting-time\": 1 }"
          check_service 90
          python -m pytest -sv test_max_concurrency.py || TEST_EXIT_CODE=1

          curl -X POST http://0.0.0.0:${FLASK_PORT}/switch \
            -H "Content-Type: application/json" \
            -d "{ \"--model\": \"/MODELDATA/ERNIE-4.5-0.3B-Paddle\", \"--max-concurrency\": 5000, \"--max-waiting-time\": 1 }"
          check_service 90
          python -m pytest -sv test_max_waiting_time.py || TEST_EXIT_CODE=1

          curl -X POST http://0.0.0.0:${FLASK_PORT}/switch \
            -H "Content-Type: application/json" \
            -d "{\"--model\": \"/MODELDATA/ernie-4_5-21b-a3b-bf16-paddle\", \"--config\": \"ernie45t_21b_sot_wint4.yaml\", \"--enable-logprob\": \"False\"}"
          check_service 360
          export TEMPLATE=TOKEN_NORMAL
          python -m pytest -sv test_seed_usage.py -k "not test_seed_stream" || TEST_EXIT_CODE=1

          curl -X POST http://0.0.0.0:${FLASK_PORT}/switch \
            -H "Content-Type: application/json" \
            -d "{\"--model\": \"/MODELDATA/ernie-4_5-21b-a3b-bf16-paddle\", \"--config\": \"ernie45t_21b_cinn_wint4.yaml\", \"--enable-logprob\": \"False\"}"
          check_service 360
          export TEMPLATE=TOKEN_NORMAL
          python -m pytest -sv test_seed_usage.py -k "not test_seed_stream" || TEST_EXIT_CODE=1

          export TEMPLATE=TOKEN_NORMAL
          curl -X POST http://0.0.0.0:${FLASK_PORT}/switch \
            -H "Content-Type: application/json" \
            -d "{\"--model\": \"/MODELDATA/ERNIE-4.5-VL-28B-A3B-Thinking\", \"--reasoning-parser\": \"ernie-45-vl-thinking\", \"--tool-call-parser\": \"ernie-45-vl-thinking\", \"--tensor-parallel-size\": 1, \"--quantization\": \"wint4\", \"--max-model-len\": 131072, \"--max-num-seqs\": 32, \"--no-enable-prefix-caching\": true}"
          check_service 180
          python -m pytest -sv test_prompt_ids.py || TEST_EXIT_CODE=1

          popd
          echo "TEST_EXIT_CODE=${TEST_EXIT_CODE}" >> /workspace/FastDeploy/exit_code.env
          '
          if [ -f ./FastDeploy/exit_code.env ]; then
            source ./FastDeploy/exit_code.env
            cat ./FastDeploy/exit_code.env >> $GITHUB_ENV
          fi
          echo "TEST_EXIT_CODE=${TEST_EXIT_CODE}"
          exit ${TEST_EXIT_CODE}

      - name: Terminate and delete the container
        if: always()
        run: |
          set +e
          docker exec -t ${{ runner.name }} /bin/bash -c 'find /workspace -mindepth 1 -delete'
          docker rm -f ${{ runner.name }}