diff --git a/docs/usage/environment_variables.md b/docs/usage/environment_variables.md index 9353947298..043ae0b378 100644 --- a/docs/usage/environment_variables.md +++ b/docs/usage/environment_variables.md @@ -227,5 +227,8 @@ environment_variables: dict[str, Callable[[], Any]] = { # XPU MoE FFN quant type map "FD_XPU_MOE_FFN_QUANT_TYPE_MAP": lambda: os.getenv("FD_XPU_MOE_FFN_QUANT_TYPE_MAP", ""), + + # Worker process health check timeout when waiting for responses in seconds (default: 30) + "FD_WORKER_ALIVE_TIMEOUT": lambda: int(os.getenv("FD_WORKER_ALIVE_TIMEOUT", "30")), } ``` diff --git a/docs/zh/usage/environment_variables.md b/docs/zh/usage/environment_variables.md index d7455471c1..82b3004092 100644 --- a/docs/zh/usage/environment_variables.md +++ b/docs/zh/usage/environment_variables.md @@ -227,5 +227,8 @@ environment_variables: dict[str, Callable[[], Any]] = { # XPU MoE FFN 量化类型映射 "FD_XPU_MOE_FFN_QUANT_TYPE_MAP": lambda: os.getenv("FD_XPU_MOE_FFN_QUANT_TYPE_MAP", ""), + + # Worker 进程响应等待时的健康检查超时时间(秒),默认 30 秒 + "FD_WORKER_ALIVE_TIMEOUT": lambda: int(os.getenv("FD_WORKER_ALIVE_TIMEOUT", "30")), } ``` diff --git a/fastdeploy/entrypoints/openai/serving_chat.py b/fastdeploy/entrypoints/openai/serving_chat.py index 580b11199a..fa3a0f74d5 100644 --- a/fastdeploy/entrypoints/openai/serving_chat.py +++ b/fastdeploy/entrypoints/openai/serving_chat.py @@ -24,6 +24,7 @@ from typing import List, Optional import numpy as np +import fastdeploy.envs as envs import fastdeploy.metrics.trace as tracing from fastdeploy.entrypoints.openai.protocol import ( ChatCompletionRequest, @@ -266,7 +267,7 @@ class OpenAIServingChat: except asyncio.TimeoutError: current_waiting_time += 10 if current_waiting_time == 300: - status, msg = self.engine_client.check_health() + status, msg = self.engine_client.check_health(time_interval_threashold=envs.FD_WORKER_ALIVE_TIMEOUT) if not status: if choices: chunk.choices = choices @@ -576,7 +577,7 @@ class OpenAIServingChat: except asyncio.TimeoutError: current_waiting_time += 10 if current_waiting_time == 300: - status, msg = self.engine_client.check_health() + status, msg = self.engine_client.check_health(time_interval_threashold=envs.FD_WORKER_ALIVE_TIMEOUT) if not status: raise ValueError(f"Engine is not healthy: {msg}") else: diff --git a/fastdeploy/entrypoints/openai/serving_completion.py b/fastdeploy/entrypoints/openai/serving_completion.py index fd86fe273c..a810ee6d4e 100644 --- a/fastdeploy/entrypoints/openai/serving_completion.py +++ b/fastdeploy/entrypoints/openai/serving_completion.py @@ -25,6 +25,7 @@ from typing import List, Optional import numpy as np +import fastdeploy.envs as envs import fastdeploy.metrics.trace as tracing from fastdeploy.engine.request import RequestOutput from fastdeploy.entrypoints.openai.protocol import ( @@ -284,7 +285,7 @@ class OpenAIServingCompletion: except asyncio.TimeoutError: current_waiting_time += 10 if current_waiting_time == 300: - status, msg = self.engine_client.check_health() + status, msg = self.engine_client.check_health(time_interval_threashold=envs.FD_WORKER_ALIVE_TIMEOUT) if not status: raise ValueError(f"Engine is not healthy: {msg}") else: @@ -454,7 +455,7 @@ class OpenAIServingCompletion: except asyncio.TimeoutError: current_waiting_time += 10 if current_waiting_time == 300: - status, msg = self.engine_client.check_health() + status, msg = self.engine_client.check_health(time_interval_threashold=envs.FD_WORKER_ALIVE_TIMEOUT) if not status: raise ValueError(f"Engine is not healthy: {msg}") else: diff --git a/fastdeploy/envs.py b/fastdeploy/envs.py index 15a64cda77..67ff60c5f5 100644 --- a/fastdeploy/envs.py +++ b/fastdeploy/envs.py @@ -159,6 +159,8 @@ environment_variables: dict[str, Callable[[], Any]] = { "FD_OTLP_EXPORTER_MAX_EXPORT_BATCH_SIZE": lambda: int(os.getenv("FD_OTLP_EXPORTER_MAX_EXPORT_BATCH_SIZE", "64")), "FD_TOKEN_PROCESSOR_HEALTH_TIMEOUT": lambda: int(os.getenv("FD_TOKEN_PROCESSOR_HEALTH_TIMEOUT", "120")), "FD_XPU_MOE_FFN_QUANT_TYPE_MAP": lambda: os.getenv("FD_XPU_MOE_FFN_QUANT_TYPE_MAP", ""), + # Timeout for worker process health check in seconds + "FD_WORKER_ALIVE_TIMEOUT": lambda: int(os.getenv("FD_WORKER_ALIVE_TIMEOUT", "30")), }