fix tokenizer oom (#6324)

fastdeploy/input/ernie4_5_vl_processor/process.py

@@ -339,7 +339,8 @@ class DataProcessor:
 
     def _add_text(self, tokens, outputs: Dict) -> None:
         if isinstance(tokens, str):
-            tokens = self.tokenizer.encode(tokens, add_special_tokens=False)["input_ids"]
+            tokens = self.tokenizer.tokenize(tokens)
+            tokens = self.tokenizer.convert_tokens_to_ids(tokens)
         outputs["input_ids"].extend(tokens)
         outputs["token_type_ids"].extend([IDS_TYPE_FLAG["text"]] * len(tokens))
 

tests/input/test_ernie_vl_processor.py

@@ -1,4 +1,4 @@
-import unittest
+import unittest
 from unittest.mock import MagicMock, patch
 
 from fastdeploy.input.ernie4_5_vl_processor import Ernie4_5_VLProcessor