enhance eos_tokens (#3274)

* enhance eos_tokens

* update

* update
This commit is contained in:
Yuanle Liu
2025-08-11 14:47:52 +08:00
committed by GitHub
parent 21caa63794
commit 9571c458f0
11 changed files with 75 additions and 93 deletions
+1 -1
View File
@@ -38,7 +38,7 @@ environment_variables: dict[str, Callable[[], Any]] = {
# Whether to use HuggingFace tokenizer (0 or 1)
"FD_USE_HF_TOKENIZER":
lambda: os.getenv("FD_USE_HF_TOKENIZER", 0),
lambda: bool(int(os.getenv("FD_USE_HF_TOKENIZER", 0))),
# ZMQ send high-water mark (HWM) during initialization
"FD_ZMQ_SNDHWM":