[CI] Optimize CI: add timeout and cancel on PR close (#6933)

.github/workflows/cancel_ci_xpu.yml

@@ -0,0 +1,20 @@
+name: CI_XPU
+
+on:
+  pull_request:
+    types: [closed]
+    branches: [develop, release/**]
+permissions: read-all
+
+concurrency:
+  group: ${{ github.event.pull_request.number }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+jobs:
+  cancel:
+    name: Cancel Coverage for ${{ github.event.pull_request.number }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Cancel Coverage
+        run: |
+          exit 0

.github/workflows/cancel_pr_build_and_test.yml

@@ -0,0 +1,19 @@
+name: PR Build and Test
+on:
+  pull_request:
+    types: [closed]
+    branches: [develop, release/**]
+permissions: read-all
+
+concurrency:
+  group: ${{ github.event.pull_request.number }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+jobs:
+  cancel:
+    name: Cancel PR Build and Test for ${{ github.event.pull_request.number }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Cancel PR Build and Test
+        run: |
+          exit 0

scripts/run_gpu_4cards.sh

@@ -20,14 +20,22 @@ fi
 
 for test_file in "${test_files[@]}"; do
     echo "------------------------------------------------------------"
-    echo "Running pytest: ${test_file}"
+    echo "Running pytest on ${test_file}"
     echo "------------------------------------------------------------"
     # Clean up previous logs
     rm -rf "${REPO_ROOT}"/log* || true
     rm -rf "${REPO_ROOT}"/*.log || true
 
-    if ! python -m pytest -sv --tb=short "${test_file}"; then
+    timeout 600 python -m pytest -sv --tb=short "${test_file}"
+    exit_code=$?
+
+    if [ $exit_code -ne 0 ]; then
+        if [ $exit_code -eq 124 ]; then
+            echo "Pytest timeout (10 min) for: ${test_file}"
+        else
             echo "Pytest failed for: ${test_file}"
+        fi
+
         echo "${test_file}" >> "${FAILED_CASE_FILE}"
         FAILED_COUNT=$((FAILED_COUNT + 1))
 

scripts/run_pre_ce.sh

@@ -32,6 +32,13 @@ for subdir in "$run_path"*/; do
                 ps -ef | grep "${FD_ENGINE_QUEUE_PORT}" | grep -v grep | awk '{print $2}' | xargs -r kill -9
 
                 if [ $exit_code -ne 0 ]; then
+                    if [ -d "${subdir%/}/log" ]; then
+                        echo ">>> grep error in ${subdir%/}/log/"
+                        grep -Rni --color=auto "error" "${subdir%/}/log/" || true
+                    else
+                        echo "${subdir%/}/log directory not found"
+                    fi
+
                     if [ -f "${subdir%/}/log/workerlog.0" ]; then
                         echo "---------------- log/workerlog.0 -------------------"
                         cat "${subdir%/}/log/workerlog.0"

tests/ce/server/core/utils.py

@@ -26,7 +26,7 @@ def build_request_payload(template_name: str, case_data: dict) -> dict:
     return final_payload
 
 
-def send_request(url, payload, timeout=600, stream=False):
+def send_request(url, payload, timeout=60, stream=False):
     """
     向指定URL发送POST请求，并返回响应结果。