[CI] Optimize port cleanup logic (#4860)

tests/e2e/test_ernie_03b_pd_multi_node.py

@@ -108,30 +108,71 @@ def get_registered_number(router_url) -> list:
 def kill_process_on_port(port: int):
     """
     Kill processes that are listening on the given port.
-    Uses `lsof` to find process ids and sends SIGKILL.
+    Uses multiple methods to ensure thorough cleanup.
     """
+    current_pid = os.getpid()
+    parent_pid = os.getppid()
+
+    # Method 1: Use lsof to find processes
     try:
         output = subprocess.check_output(f"lsof -i:{port} -t", shell=True).decode().strip()
-        current_pid = os.getpid()
-        parent_pid = os.getppid()
         for pid in output.splitlines():
             pid = int(pid)
             if pid in (current_pid, parent_pid):
                 print(f"Skip killing current process (pid={pid}) on port {port}")
                 continue
-            os.kill(pid, signal.SIGKILL)
-            print(f"Killed process on port {port}, pid={pid}")
+            try:
+                # First try SIGTERM for graceful shutdown
+                os.kill(pid, signal.SIGTERM)
+                time.sleep(1)
+                # Then SIGKILL if still running
+                os.kill(pid, signal.SIGKILL)
+                print(f"Killed process on port {port}, pid={pid}")
+            except ProcessLookupError:
+                pass  # Process already terminated
     except subprocess.CalledProcessError:
         pass
 
+    # Method 2: Use netstat and fuser as backup
+    try:
+        # Find processes using netstat and awk
+        cmd = f"netstat -tulpn 2>/dev/null | grep :{port} | awk '{{print $7}}' | cut -d'/' -f1"
+        output = subprocess.check_output(cmd, shell=True).decode().strip()
+        for pid in output.splitlines():
+            if pid and pid.isdigit():
+                pid = int(pid)
+                if pid in (current_pid, parent_pid):
+                    continue
+                try:
+                    os.kill(pid, signal.SIGKILL)
+                    print(f"Killed process (netstat) on port {port}, pid={pid}")
+                except ProcessLookupError:
+                    pass
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        pass
+
+    # Method 3: Use fuser if available
+    try:
+        subprocess.run(f"fuser -k {port}/tcp", shell=True, timeout=5)
+    except (subprocess.TimeoutExpired, subprocess.CalledProcessError, FileNotFoundError):
+        pass
+
 
 def clean_ports():
     """
     Kill all processes occupying the ports listed in PORTS_TO_CLEAN.
     """
+    print(f"Cleaning ports: {PORTS_TO_CLEAN}")
     for port in PORTS_TO_CLEAN:
         kill_process_on_port(port)
+
+    # Double check and retry if ports are still in use
     time.sleep(2)
+    for port in PORTS_TO_CLEAN:
+        if is_port_open("127.0.0.1", port, timeout=0.1):
+            print(f"Port {port} still in use, retrying cleanup...")
+            kill_process_on_port(port)
+            time.sleep(1)
 
 
 @pytest.fixture(scope="session", autouse=True)