apps/FastDeploy
1
0
Fork 0
mirror of https://github.com/PaddlePaddle/FastDeploy.git synced 2026-04-23 00:17:25 +08:00
Code Issues Actions 19 Packages Projects Releases Wiki Activity

[Optimization]Merge Text processor (#7030)

Browse Source 
* merge text processor

* update

* fix unit test

* merge messages2ids

* fix unit test

* 删除重复代码

* remove redundant code

* delete code

* fix unit test
This commit is contained in:
luukunn
2026-03-30 15:02:35 +08:00
committed by GitHub
parent 1670b011a5
commit b9f8873367
11 changed files with 765 additions and 1072 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fastdeploy/input/preprocess.py
+9 -12
View File
@@ -82,28 +82,25 @@ class InputPreprocessor:
except Exception as e:
logger.info(f"Plugin input processor not available ({e}), using built-in processor")
if not self.model_config.enable_mm:
if not ErnieArchitectures.contains_ernie_arch(architecture):
if not envs.ENABLE_V1_DATA_PROCESSOR:
from fastdeploy.input.text_processor import DataProcessor
else:
from fastdeploy.input.v1.text_processor import DataProcessor
if not envs.ENABLE_V1_DATA_PROCESSOR:
from fastdeploy.input.text_processor import TextProcessor
self.processor = DataProcessor(
tokenizer_type = "ernie4_5" if ErnieArchitectures.contains_ernie_arch(architecture) else "auto"
self.processor = TextProcessor(
model_name_or_path=self.model_name_or_path,
tokenizer_type=tokenizer_type,
reasoning_parser_obj=reasoning_parser_obj,
tool_parser_obj=tool_parser_obj,
)
else:
if not envs.ENABLE_V1_DATA_PROCESSOR:
from fastdeploy.input.ernie4_5_processor import (
Ernie4_5Processor,
)
if not ErnieArchitectures.contains_ernie_arch(architecture):
from fastdeploy.input.v1.text_processor import DataProcessor
else:
from fastdeploy.input.v1.ernie4_5_processor import (
Ernie4_5Processor,
Ernie4_5Processor as DataProcessor,
)
self.processor = Ernie4_5Processor(
self.processor = DataProcessor(
model_name_or_path=self.model_name_or_path,
reasoning_parser_obj=reasoning_parser_obj,
tool_parser_obj=tool_parser_obj,