mirror of
https://github.com/PaddlePaddle/FastDeploy.git
synced 2026-04-23 00:17:25 +08:00
YuBaoku
237 lines
9.3 KiB
YAML
237 lines
9.3 KiB
YAML
name: GOLANG_ROUTER Tests
|
|
description: "Run FastDeploy golang_router tests"
|
|
|
|
on:
|
|
workflow_call:
|
|
inputs:
|
|
DOCKER_IMAGE:
|
|
description: "Build Images"
|
|
required: true
|
|
type: string
|
|
default: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-paddle-dev"
|
|
FASTDEPLOY_ARCHIVE_URL:
|
|
description: "URL of the compressed FastDeploy code archive."
|
|
required: true
|
|
type: string
|
|
FASTDEPLOY_WHEEL_URL:
|
|
description: "URL of the FastDeploy Wheel."
|
|
required: true
|
|
type: string
|
|
FASTDEPLOY_ROUTER_URL:
|
|
description: "URL of the FastDeploy Router"
|
|
required: true
|
|
type: string
|
|
CACHE_DIR:
|
|
description: "Cache Dir Use"
|
|
required: false
|
|
type: string
|
|
default: ""
|
|
MODEL_CACHE_DIR:
|
|
description: "Cache Dir Use"
|
|
required: false
|
|
type: string
|
|
default: ""
|
|
secrets:
|
|
github-token:
|
|
required: true
|
|
|
|
jobs:
|
|
run_golang_router_tests:
|
|
runs-on: [self-hosted, GPU-h20-2Cards]
|
|
timeout-minutes: 30
|
|
steps:
|
|
- name: Code Prepare
|
|
shell: bash
|
|
env:
|
|
docker_image: ${{ inputs.DOCKER_IMAGE }}
|
|
fd_archive_url: ${{ inputs.FASTDEPLOY_ARCHIVE_URL }}
|
|
run: |
|
|
set -x
|
|
REPO="https://github.com/${{ github.repository }}.git"
|
|
FULL_REPO="${{ github.repository }}"
|
|
REPO_NAME="${FULL_REPO##*/}"
|
|
BASE_BRANCH="${{ github.base_ref }}"
|
|
docker pull ${docker_image}
|
|
# Clean the repository directory before starting
|
|
docker run --rm --net=host -v $(pwd):/workspace -w /workspace \
|
|
-e "REPO_NAME=${REPO_NAME}" \
|
|
${docker_image} /bin/bash -c '
|
|
CLEAN_RETRIES=3
|
|
CLEAN_COUNT=0
|
|
|
|
while [ $CLEAN_COUNT -lt $CLEAN_RETRIES ]; do
|
|
echo "Attempt $((CLEAN_COUNT+1)) to remove ${REPO_NAME}* ..."
|
|
rm -rf "${REPO_NAME}"* || true
|
|
sleep 2
|
|
|
|
# Check if anything matching ${REPO_NAME}* still exists
|
|
if ! ls "${REPO_NAME}"* >/dev/null 2>&1; then
|
|
echo "All ${REPO_NAME}* removed successfully"
|
|
break
|
|
fi
|
|
|
|
CLEAN_COUNT=$((CLEAN_COUNT + 1))
|
|
done
|
|
|
|
if ls "${REPO_NAME}"* >/dev/null 2>&1; then
|
|
echo "ERROR: Failed to clean ${REPO_NAME}* after multiple attempts"
|
|
ls -ld "${REPO_NAME}"*
|
|
echo "Attempting force cleanup with find..."
|
|
find /workspace -mindepth 1 -maxdepth 1 -name "${REPO_NAME}*" -type d -exec chmod -R u+rwx {} \; -exec rm -rf {} + 2>/dev/null || true
|
|
if ls "${REPO_NAME}"* >/dev/null 2>&1; then
|
|
echo "ERROR: Force cleanup still failed"
|
|
exit 1
|
|
else
|
|
echo "Force cleanup succeeded"
|
|
fi
|
|
fi
|
|
'
|
|
|
|
wget -q --no-proxy ${fd_archive_url} || {
|
|
echo "ERROR: Failed to download archive from ${fd_archive_url}"
|
|
exit 1
|
|
}
|
|
|
|
tar --no-same-owner -xf FastDeploy.tar.gz || {
|
|
echo "ERROR: Failed to extract archive"
|
|
exit 1
|
|
}
|
|
|
|
rm -rf FastDeploy.tar.gz
|
|
cd FastDeploy
|
|
git config --global user.name "FastDeployCI"
|
|
git config --global user.email "fastdeploy_ci@example.com"
|
|
git log -n 3 --oneline
|
|
|
|
- name: Run Golang_Router Tests
|
|
shell: bash
|
|
env:
|
|
docker_image: ${{ inputs.DOCKER_IMAGE }}
|
|
fd_wheel_url: ${{ inputs.FASTDEPLOY_WHEEL_URL }}
|
|
fd_router_url: ${{ inputs.FASTDEPLOY_ROUTER_URL }}
|
|
CACHE_DIR: ${{ inputs.CACHE_DIR }}
|
|
BASE_REF: ${{ github.event.pull_request.base.ref }}
|
|
MODEL_CACHE_DIR: ${{ inputs.MODEL_CACHE_DIR }}
|
|
IS_PR: ${{ github.event_name == 'pull_request' }}
|
|
run: |
|
|
if [[ "$IS_PR" == "true" ]]; then
|
|
echo "Running on PR"
|
|
else
|
|
echo "Not a PR"
|
|
fi
|
|
runner_name="${{ runner.name }}"
|
|
CARD_ID=$(echo "${runner_name}" | awk -F'-' '{print $NF}')
|
|
DEVICES=$(echo "$CARD_ID" | fold -w1 | paste -sd,)
|
|
DEVICE_PORT=$(echo "$DEVICES" | cut -d',' -f1)
|
|
|
|
FLASK_PORT=$((8068 + DEVICE_PORT * 100))
|
|
FD_API_PORT=$((8088 + DEVICE_PORT * 100))
|
|
FD_ENGINE_QUEUE_PORT=$((8058 + DEVICE_PORT * 100))
|
|
FD_METRICS_PORT=$((8078 + DEVICE_PORT * 100))
|
|
FD_CACHE_QUEUE_PORT=$((8098 + DEVICE_PORT * 100))
|
|
FD_ROUTER_PORT=$((8048 + DEVICE_PORT * 100))
|
|
FD_CONNECTOR_PORT=$((8038 + DEVICE_PORT * 100))
|
|
FD_RDMA_PORT=$((8028 + DEVICE_PORT * 100))
|
|
echo "Test ENV Parameter:"
|
|
echo "========================================================="
|
|
echo "FLASK_PORT=${FLASK_PORT}"
|
|
echo "FD_API_PORT=${FD_API_PORT}"
|
|
echo "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}"
|
|
echo "FD_METRICS_PORT=${FD_METRICS_PORT}"
|
|
echo "FD_CACHE_QUEUE_PORT=${FD_CACHE_QUEUE_PORT}"
|
|
echo "FD_ROUTER_PORT=${FD_ROUTER_PORT}"
|
|
echo "FD_CONNECTOR_PORT=${FD_CONNECTOR_PORT}"
|
|
echo "FD_RDMA_PORT=${FD_RDMA_PORT}"
|
|
echo "DEVICES=${DEVICES}"
|
|
echo "========================================================="
|
|
|
|
CACHE_DIR="${CACHE_DIR:-$(dirname "$(dirname "${{ github.workspace }}")")}"
|
|
echo "CACHE_DIR is set to ${CACHE_DIR}"
|
|
if [ ! -f "${CACHE_DIR}/gitconfig" ]; then
|
|
touch "${CACHE_DIR}/gitconfig"
|
|
fi
|
|
|
|
PORTS=($FLASK_PORT $FD_API_PORT $FD_ENGINE_QUEUE_PORT $FD_METRICS_PORT $FD_CACHE_QUEUE_PORT)
|
|
LOG_FILE="./port_cleanup_$(date +%Y%m%d_%H%M%S).log"
|
|
echo "==== LOG_FILE is ${LOG_FILE} ===="
|
|
|
|
echo "==== PORT CLEAN BEFORE TASK RUN ====" | tee -a $LOG_FILE
|
|
|
|
for port in "${PORTS[@]}"; do
|
|
PIDS=$(lsof -t -i :$port || true)
|
|
if [ -n "$PIDS" ]; then
|
|
echo "Port $port is occupied by PID(s): $PIDS" | tee -a $LOG_FILE
|
|
echo "$PIDS" | xargs -r kill -9
|
|
echo "Port $port cleared" | tee -a $LOG_FILE
|
|
else
|
|
echo "Port $port is free" | tee -a $LOG_FILE
|
|
fi
|
|
done
|
|
|
|
echo "==== PORT CLEAN COMPLETE ====" | tee -a $LOG_FILE
|
|
|
|
echo "========================================================="
|
|
echo "Ensuring no stale container named ${runner_name} ..."
|
|
if [ "$(docker ps -a -q -f name=${runner_name})" ]; then
|
|
echo "Removing stale container: ${runner_name}"
|
|
docker rm -f ${runner_name} || true
|
|
fi
|
|
|
|
export RDMA_DEVICES=$(find /dev/infiniband/uverbs* -maxdepth 1 -not -type d | xargs -I{} echo '--device {}:{}')
|
|
|
|
docker run --rm --net=host \
|
|
--name ${runner_name} \
|
|
--cap-add=SYS_PTRACE --cap-add=IPC_LOCK \
|
|
--shm-size=64G \
|
|
${RDMA_DEVICES} \
|
|
--device=/dev/infiniband/rdma_cm \
|
|
--ulimit memlock=-1:-1 \
|
|
-v $(pwd):/workspace -w /workspace \
|
|
-v "${CACHE_DIR}/gitconfig:/etc/gitconfig:ro" \
|
|
-v "${CACHE_DIR}/.cache:/root/.cache" \
|
|
-v "${CACHE_DIR}/ConfigDir:/root/.config" \
|
|
-v "${MODEL_CACHE_DIR}:/ModelData:ro" \
|
|
-e "MODEL_PATH=/ModelData" \
|
|
-e "FD_API_PORT=${FD_API_PORT}" \
|
|
-e "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}" \
|
|
-e "FD_METRICS_PORT=${FD_METRICS_PORT}" \
|
|
-e "FLASK_PORT=${FLASK_PORT}" \
|
|
-e "FD_CACHE_QUEUE_PORT=${FD_CACHE_QUEUE_PORT}" \
|
|
-e "FD_ROUTER_PORT=${FD_ROUTER_PORT}" \
|
|
-e "FD_CONNECTOR_PORT=${FD_CONNECTOR_PORT}" \
|
|
-e "FD_RDMA_PORT=${FD_RDMA_PORT}" \
|
|
-e "CLEAN_CUDA=1" \
|
|
-e TZ="Asia/Shanghai" \
|
|
-e "fd_wheel_url=${fd_wheel_url}" \
|
|
-e "fd_router_url=${fd_router_url}" \
|
|
-e "BASE_REF=${BASE_REF}" \
|
|
-e "IS_PR=${IS_PR}" \
|
|
-e "no_proxy=localhost,127.0.0.1,0.0.0.0,bcebos.com,.bcebos.com,bj.bcebos.com,su.bcebos.com,paddle-ci.gz.bcebos.com,apiin.im.baidu.com,baidu-int.com,.baidu.com,aliyun.com,gitee.com,pypi.tuna.tsinghua.edu.cn,.tuna.tsinghua.edu.cn" \
|
|
--gpus "\"device=${DEVICES}\"" ${docker_image} /bin/bash -c '
|
|
|
|
git config --global --add safe.directory /workspace/FastDeploy
|
|
cd FastDeploy
|
|
|
|
python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
|
|
pip config set global.extra-index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
|
|
|
|
python -m pip install -r scripts/unittest_requirement.txt
|
|
python -m pip install ${fd_wheel_url}
|
|
rm -rf fastdeploy
|
|
python -m pip install ${fd_wheel_url} --no-deps --target=/workspace/FastDeploy
|
|
export PYTHONPATH=/workspace/FastDeploy/
|
|
|
|
# download fd-router binary
|
|
wget -q --no-proxy ${fd_router_url} -O /usr/local/bin/fd-router
|
|
chmod +x /usr/local/bin/fd-router
|
|
|
|
bash scripts/run_golang_router.sh
|
|
'
|
|
|
|
- name: Terminate and delete the container
|
|
if: always()
|
|
run: |
|
|
set +e
|
|
docker exec -t ${{ runner.name }} /bin/bash -c 'find /workspace -mindepth 1 -delete'
|
|
docker rm -f ${{ runner.name }}
|