apps/FastDeploy
1
0
Fork 0
mirror of https://github.com/PaddlePaddle/FastDeploy.git synced 2026-04-23 17:11:21 +08:00
Code Issues Actions 10 Packages Projects Releases Wiki Activity

[Bug Fix] fix tokenizer oom (#6287)

Browse Source 
* fix tokenizer oom

* fix unit test
This commit is contained in:
ApplEOFDiscord
2026-02-03 11:27:11 +08:00
committed by GitHub
parent fb374238e1
commit 6563b8307c
2 changed files with 4 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fastdeploy/input/ernie4_5_vl_processor/process.py
+2 -1
View File
@@ -480,7 +480,8 @@ class DataProcessor(MMBaseDataProcessor):
def _add_text(self, tokens, outputs: Dict) -> None:
if isinstance(tokens, str):
tokens = self.tokenizer.encode(tokens, add_special_tokens=False)["input_ids"]
tokens = self.tokenizer.tokenize(tokens)
tokens = self.tokenizer.convert_tokens_to_ids(tokens)
outputs["input_ids"].extend(tokens)
outputs["token_type_ids"].extend([IDS_TYPE_FLAG["text"]] * len(tokens))