apps/FastDeploy
1
0
Fork 0
mirror of https://github.com/PaddlePaddle/FastDeploy.git synced 2026-04-23 00:17:25 +08:00
Code Issues Actions 19 Packages Projects Releases Wiki Activity

[BugFix][Scheduler]Fix FD_DISABLE_CHUNKED_PREFILL max_num_batched_tokens limit (#7407)

Browse Source 
* fix FD_DISABLE_CHUNKED_PREFILL max_num_batched_tokens=max_model_len

* fix FD_DISABLE_CHUNKED_PREFILL max_num_batched_tokens=max_model_len
This commit is contained in:
AIbin
2026-04-15 15:55:11 +08:00
committed by GitHub
parent 5e54770b2e
commit 8eebbcaf15
2 changed files with 9 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fastdeploy/config.py
+4 -1
View File
@@ -2064,7 +2064,10 @@ class FDConfig:
if self.scheduler_config.max_num_batched_tokens is None: if self.scheduler_config.max_num_batched_tokens is None:
if int(envs.ENABLE_V1_KVCACHE_SCHEDULER): if int(envs.ENABLE_V1_KVCACHE_SCHEDULER):
self.scheduler_config.max_num_batched_tokens = 8192 # if set to max_model_len, it's easy to be OOM if int(envs.FD_DISABLE_CHUNKED_PREFILL):
self.scheduler_config.max_num_batched_tokens = self.model_config.max_model_len
else:
self.scheduler_config.max_num_batched_tokens = 8192 # if set to max_model_len, it's easy to be OOM
else: else:
if self.cache_config.enable_chunked_prefill: if self.cache_config.enable_chunked_prefill:
self.scheduler_config.max_num_batched_tokens = 2048 self.scheduler_config.max_num_batched_tokens = 2048
fastdeploy/engine/args_utils.py
+5 -1
View File
@@ -1498,7 +1498,11 @@ class EngineArgs:
if self.max_num_batched_tokens is None: if self.max_num_batched_tokens is None:
if int(envs.ENABLE_V1_KVCACHE_SCHEDULER): if int(envs.ENABLE_V1_KVCACHE_SCHEDULER):
if current_platform.is_maca() or current_platform.is_iluvatar(): if (
int(envs.FD_DISABLE_CHUNKED_PREFILL)
or current_platform.is_maca()
or current_platform.is_iluvatar()
):
self.max_num_batched_tokens = self.max_model_len self.max_num_batched_tokens = self.max_model_len
else: else:
self.max_num_batched_tokens = 8192 # if set to max_model_len, it's easy to be OOM self.max_num_batched_tokens = 8192 # if set to max_model_len, it's easy to be OOM