[Iluvatar GPU] Adapt VL model (#4313)

fastdeploy/model_executor/models/ernie4_5_vl/dfnrope/modeling.py

@@ -35,6 +35,7 @@ from paddleformers.transformers.model_utils import PretrainedModel
 
 from fastdeploy.model_executor.layers.utils import divide, get_tensor
 from fastdeploy.model_executor.utils import set_weight_attrs
+from fastdeploy.platforms import current_platform
 
 from .activation import ACT2FN
 from .configuration import DFNRopeVisionTransformerConfig
@@ -174,7 +175,7 @@ class VisionFlashAttention2(nn.Layer):
                 mp_group=fleet.get_hybrid_communicate_group().get_model_parallel_group(),
                 weight_attr=None,
                 has_bias=True,
-                fuse_matmul_bias=True,
+                fuse_matmul_bias=False if current_platform.is_iluvatar() else True,
                 gather_output=False,
             )
             self.proj = RowParallelLinear(

fastdeploy/model_executor/models/ernie4_5_vl/image_op.py

@@ -26,6 +26,11 @@ elif current_platform.is_xpu():
         text_image_gather_scatter,
         text_image_index_out,
     )
+elif current_platform.is_iluvatar():
+    from fastdeploy.model_executor.ops.iluvatar import (
+        text_image_gather_scatter,
+        text_image_index_out,
+    )
 else:
     raise ImportError("Unsupported platform, only support CUDA and XPU")
 

fastdeploy/model_executor/models/ernie4_5_vl/modeling_resampler.py

@@ -31,6 +31,7 @@ from fastdeploy.model_executor.models.ernie4_5_vl.dist_utils import (
     scatter_axis,
 )
 from fastdeploy.model_executor.utils import set_weight_attrs
+from fastdeploy.platforms import current_platform
 
 
 class ScatterOp(PyLayer):
@@ -172,7 +173,7 @@ class VariableResolutionResamplerModel(nn.Layer):
                         self.spatial_dim,
                         input_is_parallel=True,
                         has_bias=True,
-                        fuse_matmul_bias=True,
+                        fuse_matmul_bias=False if current_platform.is_iluvatar() else True,
                     )
                     if self.tensor_parallel_degree > 1
                     else nn.Linear(self.spatial_dim, self.spatial_dim)