init (#6642)

Co-authored-by: gongweibao <gognweibao@baidu.com>

custom_ops/gpu_ops/cutlass_kernels/w8a8/c3x/cutlass_gemm_caller.cuh

@@ -1,4 +1,5 @@
-// adapted from: https://github.com/vllm-project/vllm/blob/118ff921118cc81061a2af865a1e13840ceb6792/csrc/quantization/cutlass_w8a8/c3x/cutlass_gemm_caller.cuh
+// adapted from:
+// https://github.com/vllm-project/vllm/blob/118ff921118cc81061a2af865a1e13840ceb6792/csrc/quantization/cutlass_w8a8/c3x/cutlass_gemm_caller.cuh
 #pragma once
 
 // clang-format will break include orders
@@ -21,15 +22,16 @@
 
 namespace fastdeploy::c3x {
 
-static inline cute::Shape<int, int, int, int>
-get_problem_shape(paddle::Tensor const &a, paddle::Tensor const &b) {
+static inline cute::Shape<int, int, int, int> get_problem_shape(
+    paddle::Tensor const &a, paddle::Tensor const &b) {
   int32_t m = a.dims()[0], n = b.dims()[0], k = a.dims()[1];
   return {m, n, k, 1};
 }
 
 template <typename GemmKernel>
 void cutlass_gemm_caller(
-    phi::Place device, cute::Shape<int, int, int, int> prob_shape,
+    phi::Place device,
+    cute::Shape<int, int, int, int> prob_shape,
     typename GemmKernel::MainloopArguments mainloop_args,
     typename GemmKernel::EpilogueArguments epilogue_args,
     typename GemmKernel::TileSchedulerArguments scheduler = {}) {
@@ -57,7 +59,8 @@ void cutlass_gemm_caller(
 }
 
 template <typename Gemm, typename... EpilogueArgs>
-void cutlass_gemm_caller(paddle::Tensor &out, paddle::Tensor const &a,
+void cutlass_gemm_caller(paddle::Tensor &out,
+                         paddle::Tensor const &a,
                          paddle::Tensor const &b,
                          EpilogueArgs &&...epilogue_params) {
   using ElementAB = typename Gemm::ElementAB;
@@ -86,17 +89,20 @@ void cutlass_gemm_caller(paddle::Tensor &out, paddle::Tensor const &a,
 
   auto a_ptr = static_cast<ElementAB *>(const_cast<void *>(a.data()));
   auto b_ptr = static_cast<ElementAB *>(const_cast<void *>(b.data()));
-  typename GemmKernel::MainloopArguments mainloop_args{a_ptr, a_stride, b_ptr,
-                                                       b_stride};
+  typename GemmKernel::MainloopArguments mainloop_args{
+      a_ptr, a_stride, b_ptr, b_stride};
 
   auto c_ptr = static_cast<ElementD *>(const_cast<void *>(out.data()));
   typename GemmKernel::EpilogueArguments epilogue_args{
       Gemm::Epilogue::prepare_args(
           std::forward<EpilogueArgs>(epilogue_params)...),
-      c_ptr, c_stride, c_ptr, d_stride};
+      c_ptr,
+      c_stride,
+      c_ptr,
+      d_stride};
 
-  cutlass_gemm_caller<GemmKernel>(a.place(), prob_shape, mainloop_args,
-                                  epilogue_args);
+  cutlass_gemm_caller<GemmKernel>(
+      a.place(), prob_shape, mainloop_args, epilogue_args);
 }
 
-} // namespace fastdeploy::c3x
+}  // namespace fastdeploy::c3x

custom_ops/gpu_ops/cutlass_kernels/w8a8/c3x/scaled_mm.cuh

@@ -1,4 +1,5 @@
-// adapted from: https://github.com/vllm-project/vllm/blob/118ff921118cc81061a2af865a1e13840ceb6792/csrc/quantization/cutlass_w8a8/c3x/scaled_mm.cuh
+// adapted from:
+// https://github.com/vllm-project/vllm/blob/118ff921118cc81061a2af865a1e13840ceb6792/csrc/quantization/cutlass_w8a8/c3x/scaled_mm.cuh
 
 #pragma once
 
@@ -31,16 +32,19 @@ using namespace cute;
 
 namespace fastdeploy {
 
-template <typename ElementAB_, typename ElementD_,
-          template <typename, typename, typename> typename Epilogue_,
-          typename TileShape, typename ClusterShape, typename KernelSchedule,
+template <typename ElementAB_,
+          typename ElementD_,
+          template <typename, typename, typename>
+          typename Epilogue_,
+          typename TileShape,
+          typename ClusterShape,
+          typename KernelSchedule,
           typename EpilogueSchedule>
 struct cutlass_3x_gemm {
   using ElementAB = ElementAB_;
   using ElementD = ElementD_;
-  using ElementAcc =
-      typename std::conditional<std::is_same_v<ElementAB, int8_t>, int32_t,
-                                float>::type;
+  using ElementAcc = typename std::
+      conditional<std::is_same_v<ElementAB, int8_t>, int32_t, float>::type;
 
   using Epilogue = Epilogue_<ElementAcc, ElementD, TileShape>;
 
@@ -57,10 +61,21 @@ struct cutlass_3x_gemm {
 
   using CollectiveEpilogue =
       typename cutlass::epilogue::collective::CollectiveBuilder<
-          cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp, TileShape,
-          ClusterShape, cutlass::epilogue::collective::EpilogueTileAuto,
-          ElementAcc, float, ElementC, StrideC, AlignmentCD, ElementD, StrideD,
-          AlignmentCD, EpilogueSchedule, EVTCompute>::CollectiveOp;
+          cutlass::arch::Sm90,
+          cutlass::arch::OpClassTensorOp,
+          TileShape,
+          ClusterShape,
+          cutlass::epilogue::collective::EpilogueTileAuto,
+          ElementAcc,
+          float,
+          ElementC,
+          StrideC,
+          AlignmentCD,
+          ElementD,
+          StrideD,
+          AlignmentCD,
+          EpilogueSchedule,
+          EVTCompute>::CollectiveOp;
 
   static constexpr size_t CEStorageSize =
       sizeof(typename CollectiveEpilogue::SharedStorage);
@@ -78,16 +93,22 @@ struct cutlass_3x_gemm {
           KernelSchedule>::CollectiveOp;
   // clang-format on
 
-  using KernelType = enable_sm90_or_later<cutlass::gemm::kernel::GemmUniversal<
-      cute::Shape<int, int, int, int>, CollectiveMainloop, CollectiveEpilogue,
-      cutlass::gemm::PersistentScheduler>>;
+  using KernelType = enable_sm90_or_later<
+      cutlass::gemm::kernel::GemmUniversal<cute::Shape<int, int, int, int>,
+                                           CollectiveMainloop,
+                                           CollectiveEpilogue,
+                                           cutlass::gemm::PersistentScheduler>>;
 
   struct GemmKernel : public KernelType {};
 };
 
-template <typename ElementAB_, typename ElementD_,
-          template <typename, typename, typename> typename Epilogue_,
-          typename TileShape, typename ClusterShape, typename KernelSchedule,
+template <typename ElementAB_,
+          typename ElementD_,
+          template <typename, typename, typename>
+          typename Epilogue_,
+          typename TileShape,
+          typename ClusterShape,
+          typename KernelSchedule,
           typename EpilogueSchedule>
 struct cutlass_3x_gemm_sm100 {
   using ElementAB = ElementAB_;
@@ -108,9 +129,8 @@ struct cutlass_3x_gemm_sm100 {
   using LayoutD = cutlass::layout::RowMajor;
   static constexpr int AlignmentD = AlignmentC;
 
-  using ElementAcc =
-      typename std::conditional<std::is_same_v<ElementAB, int8_t>, int32_t,
-                                float>::type;
+  using ElementAcc = typename std::
+      conditional<std::is_same_v<ElementAB, int8_t>, int32_t, float>::type;
   using Epilogue = Epilogue_<ElementAcc, ElementD, TileShape>;
 
   // MMA type
@@ -127,23 +147,44 @@ struct cutlass_3x_gemm_sm100 {
 
   using CollectiveEpilogue =
       typename cutlass::epilogue::collective::CollectiveBuilder<
-          cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, TileShape,
-          ClusterShape, cutlass::epilogue::collective::EpilogueTileAuto,
-          ElementAccumulator, ElementCompute, ElementC, LayoutC, AlignmentC,
-          ElementD, LayoutD, AlignmentD, EpilogueSchedule,
+          cutlass::arch::Sm100,
+          cutlass::arch::OpClassTensorOp,
+          TileShape,
+          ClusterShape,
+          cutlass::epilogue::collective::EpilogueTileAuto,
+          ElementAccumulator,
+          ElementCompute,
+          ElementC,
+          LayoutC,
+          AlignmentC,
+          ElementD,
+          LayoutD,
+          AlignmentD,
+          EpilogueSchedule,
           EVTCompute>::CollectiveOp;
 
   using CollectiveMainloop =
       typename cutlass::gemm::collective::CollectiveBuilder<
-          cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, ElementAB,
-          LayoutA, AlignmentA, ElementAB, LayoutB, AlignmentB,
-          ElementAccumulator, TileShape, ClusterShape,
+          cutlass::arch::Sm100,
+          cutlass::arch::OpClassTensorOp,
+          ElementAB,
+          LayoutA,
+          AlignmentA,
+          ElementAB,
+          LayoutB,
+          AlignmentB,
+          ElementAccumulator,
+          TileShape,
+          ClusterShape,
           cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(
               sizeof(typename CollectiveEpilogue::SharedStorage))>,
           KernelSchedule>::CollectiveOp;
 
-  using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
-      Shape<int, int, int, int>, CollectiveMainloop, CollectiveEpilogue, void>;
+  using GemmKernel =
+      cutlass::gemm::kernel::GemmUniversal<Shape<int, int, int, int>,
+                                           CollectiveMainloop,
+                                           CollectiveEpilogue,
+                                           void>;
 };
 
-} // namespace fastdeploy
+}  // namespace fastdeploy

custom_ops/gpu_ops/cutlass_kernels/w8a8/c3x/scaled_mm_azp_sm90_int8.cu

@@ -1,4 +1,5 @@
-// adapted from: https://github.com/vllm-project/vllm/blob/118ff921118cc81061a2af865a1e13840ceb6792/csrc/quantization/cutlass_w8a8/c3x/scaled_mm_azp_sm90_int8.cu
+// adapted from:
+// https://github.com/vllm-project/vllm/blob/118ff921118cc81061a2af865a1e13840ceb6792/csrc/quantization/cutlass_w8a8/c3x/scaled_mm_azp_sm90_int8.cu
 
 // clang-format will break include orders
 // clang-format off
@@ -10,18 +11,22 @@
 namespace fastdeploy {
 
 void cutlass_scaled_mm_azp_sm90_int8(
-    paddle::Tensor &out, paddle::Tensor const &a, paddle::Tensor const &b,
-    paddle::Tensor const &a_scales, paddle::Tensor const &b_scales,
-    paddle::Tensor const &azp_adj, paddle::optional<paddle::Tensor> const &azp,
+    paddle::Tensor &out,
+    paddle::Tensor const &a,
+    paddle::Tensor const &b,
+    paddle::Tensor const &a_scales,
+    paddle::Tensor const &b_scales,
+    paddle::Tensor const &azp_adj,
+    paddle::optional<paddle::Tensor> const &azp,
     paddle::optional<paddle::Tensor> const &bias) {
   if (azp) {
     return cutlass_scaled_mm_sm90_int8_epilogue<
-        c3x::ScaledEpilogueBiasAzpToken>(out, a, b, a_scales, b_scales, azp_adj,
-                                         *azp, bias);
+        c3x::ScaledEpilogueBiasAzpToken>(
+        out, a, b, a_scales, b_scales, azp_adj, *azp, bias);
   } else {
     return cutlass_scaled_mm_sm90_int8_epilogue<c3x::ScaledEpilogueBiasAzp>(
         out, a, b, a_scales, b_scales, azp_adj, bias);
   }
 }
 
-} // namespace fastdeploy
+}  // namespace fastdeploy

custom_ops/gpu_ops/cutlass_kernels/w8a8/c3x/scaled_mm_helper.hpp

@@ -1,34 +1,38 @@
-// adapted from: https://github.com/vllm-project/vllm/blob/118ff921118cc81061a2af865a1e13840ceb6792/csrc/quantization/cutlass_w8a8/c3x/scaled_mm_helper.hpp
+// adapted from:
+// https://github.com/vllm-project/vllm/blob/118ff921118cc81061a2af865a1e13840ceb6792/csrc/quantization/cutlass_w8a8/c3x/scaled_mm_helper.hpp
 
 #include "helper.h"
 
 template <typename Fp8Func, typename Int8Func>
-void dispatch_scaled_mm(paddle::Tensor &c, paddle::Tensor const &a,
-                        paddle::Tensor const &b, paddle::Tensor const &a_scales,
+void dispatch_scaled_mm(paddle::Tensor &c,
+                        paddle::Tensor const &a,
+                        paddle::Tensor const &b,
+                        paddle::Tensor const &a_scales,
                         paddle::Tensor const &b_scales,
                         paddle::optional<paddle::Tensor> const &bias,
-                        Fp8Func fp8_func, Int8Func int8_func) {
-    PD_CHECK(a_scales.dtype() == paddle::DataType::FLOAT32);
-    PD_CHECK(b_scales.dtype() == paddle::DataType::FLOAT32);
+                        Fp8Func fp8_func,
+                        Int8Func int8_func) {
+  PD_CHECK(a_scales.dtype() == paddle::DataType::FLOAT32);
+  PD_CHECK(b_scales.dtype() == paddle::DataType::FLOAT32);
 
-    int M = a.dims()[0], N = b.dims()[0], K = a.dims()[1];
+  int M = a.dims()[0], N = b.dims()[0], K = a.dims()[1];
 
-    if ((a_scales.numel() == 1 || a_scales.numel() == a.dims()[0]) &&
-        (b_scales.numel() == 1 || b_scales.numel() == b.dims()[0])) {
-      // Standard per-tensor/per-token/per-channel scaling
-      PD_CHECK(a_scales.is_contiguous() && b_scales.is_contiguous());
-      if (a.dtype() == phi::DataType::FLOAT8_E4M3FN) {
-        fp8_func(c, a, b, a_scales, b_scales, bias);
-      } else {
-        PD_CHECK(a.dtype() == paddle::DataType::INT8);
-        if constexpr (!std::is_same_v<Int8Func, std::nullptr_t>) {
-          int8_func(c, a, b, a_scales, b_scales, bias);
-        } else {
-          PD_CHECK(false, "Int8 not supported for this architecture");
-        }
-      }
+  if ((a_scales.numel() == 1 || a_scales.numel() == a.dims()[0]) &&
+      (b_scales.numel() == 1 || b_scales.numel() == b.dims()[0])) {
+    // Standard per-tensor/per-token/per-channel scaling
+    PD_CHECK(a_scales.is_contiguous() && b_scales.is_contiguous());
+    if (a.dtype() == phi::DataType::FLOAT8_E4M3FN) {
+      fp8_func(c, a, b, a_scales, b_scales, bias);
     } else {
-      PADDLE_THROW(phi::errors::Unimplemented(
-          "No kernel for this combination of input dtypes is implemented."));
+      PD_CHECK(a.dtype() == paddle::DataType::INT8);
+      if constexpr (!std::is_same_v<Int8Func, std::nullptr_t>) {
+        int8_func(c, a, b, a_scales, b_scales, bias);
+      } else {
+        PD_CHECK(false, "Int8 not supported for this architecture");
+      }
     }
+  } else {
+    PADDLE_THROW(phi::errors::Unimplemented(
+        "No kernel for this combination of input dtypes is implemented."));
+  }
 }

custom_ops/gpu_ops/cutlass_kernels/w8a8/c3x/scaled_mm_kernels.hpp

@@ -1,4 +1,5 @@
-// adapted from: https://github.com/vllm-project/vllm/blob/118ff921118cc81061a2af865a1e13840ceb6792/csrc/quantization/cutlass_w8a8/c3x/scaled_mm_kernels.hpp
+// adapted from:
+// https://github.com/vllm-project/vllm/blob/118ff921118cc81061a2af865a1e13840ceb6792/csrc/quantization/cutlass_w8a8/c3x/scaled_mm_kernels.hpp
 
 #pragma once
 
@@ -6,30 +7,35 @@
 
 namespace fastdeploy {
 
-void cutlass_scaled_mm_sm90_fp8(paddle::Tensor &out, paddle::Tensor const &a,
+void cutlass_scaled_mm_sm90_fp8(paddle::Tensor &out,
+                                paddle::Tensor const &a,
                                 paddle::Tensor const &b,
                                 paddle::Tensor const &a_scales,
                                 paddle::Tensor const &b_scales,
                                 paddle::optional<paddle::Tensor> const &bias);
 
-void cutlass_scaled_mm_sm90_int8(paddle::Tensor &out, paddle::Tensor const &a,
+void cutlass_scaled_mm_sm90_int8(paddle::Tensor &out,
+                                 paddle::Tensor const &a,
                                  paddle::Tensor const &b,
                                  paddle::Tensor const &a_scales,
                                  paddle::Tensor const &b_scales,
                                  paddle::optional<paddle::Tensor> const &bias);
 
-void cutlass_scaled_mm_azp_sm90_int8(paddle::Tensor& out, paddle::Tensor const& a,
-                                     paddle::Tensor const& b,
-                                     paddle::Tensor const& a_scales,
-                                     paddle::Tensor const& b_scales,
-                                     paddle::Tensor const& azp_adj,
-                                     paddle::optional<paddle::Tensor> const& azp,
-                                     paddle::optional<paddle::Tensor> const& bias);
+void cutlass_scaled_mm_azp_sm90_int8(
+    paddle::Tensor &out,
+    paddle::Tensor const &a,
+    paddle::Tensor const &b,
+    paddle::Tensor const &a_scales,
+    paddle::Tensor const &b_scales,
+    paddle::Tensor const &azp_adj,
+    paddle::optional<paddle::Tensor> const &azp,
+    paddle::optional<paddle::Tensor> const &bias);
 
-void cutlass_scaled_mm_sm100_fp8(paddle::Tensor &out, paddle::Tensor const &a,
+void cutlass_scaled_mm_sm100_fp8(paddle::Tensor &out,
+                                 paddle::Tensor const &a,
                                  paddle::Tensor const &b,
                                  paddle::Tensor const &a_scales,
                                  paddle::Tensor const &b_scales,
                                  paddle::optional<paddle::Tensor> const &bias);
 
-} // namespace fastdeploy
+}  // namespace fastdeploy

custom_ops/gpu_ops/cutlass_kernels/w8a8/c3x/scaled_mm_sm90_fp8.cu

@@ -1,4 +1,5 @@
-// adapted from: https://github.com/vllm-project/vllm/blob/118ff921118cc81061a2af865a1e13840ceb6792/csrc/quantization/cutlass_w8a8/c3x/scaled_mm_sm90_fp8.cu
+// adapted from:
+// https://github.com/vllm-project/vllm/blob/118ff921118cc81061a2af865a1e13840ceb6792/csrc/quantization/cutlass_w8a8/c3x/scaled_mm_sm90_fp8.cu
 
 // clang-format will break include orders
 // clang-format off
@@ -9,7 +10,8 @@
 
 namespace fastdeploy {
 
-void cutlass_scaled_mm_sm90_fp8(paddle::Tensor &out, paddle::Tensor const &a,
+void cutlass_scaled_mm_sm90_fp8(paddle::Tensor &out,
+                                paddle::Tensor const &a,
                                 paddle::Tensor const &b,
                                 paddle::Tensor const &a_scales,
                                 paddle::Tensor const &b_scales,
@@ -17,7 +19,8 @@ void cutlass_scaled_mm_sm90_fp8(paddle::Tensor &out, paddle::Tensor const &a,
   PD_CHECK(a_scales.is_contiguous() && b_scales.is_contiguous());
   if (bias) {
     PD_CHECK(bias->dtype() == out.dtype(),
-             "currently bias dtype must match output dtype ", out.dtype());
+             "currently bias dtype must match output dtype ",
+             out.dtype());
     return cutlass_scaled_mm_sm90_fp8_epilogue<c3x::ScaledEpilogueBias>(
         out, a, b, a_scales, b_scales, *bias);
   } else {
@@ -25,4 +28,4 @@ void cutlass_scaled_mm_sm90_fp8(paddle::Tensor &out, paddle::Tensor const &a,
         out, a, b, a_scales, b_scales);
   }
 }
-} // namespace fastdeploy
+}  // namespace fastdeploy

custom_ops/gpu_ops/cutlass_kernels/w8a8/c3x/scaled_mm_sm90_fp8_dispatch.cuh

@@ -1,4 +1,5 @@
-// adapted from: https://github.com/vllm-project/vllm/blob/118ff921118cc81061a2af865a1e13840ceb6792/csrc/quantization/cutlass_w8a8/c3x/scaled_mm_sm90_fp8_dispatch.cuh
+// adapted from:
+// https://github.com/vllm-project/vllm/blob/118ff921118cc81061a2af865a1e13840ceb6792/csrc/quantization/cutlass_w8a8/c3x/scaled_mm_sm90_fp8_dispatch.cuh
 
 #pragma once
 
@@ -17,8 +18,10 @@ namespace fastdeploy {
 
 using c3x::cutlass_gemm_caller;
 
-template <typename InType, typename OutType,
-          template <typename, typename, typename> typename Epilogue>
+template <typename InType,
+          typename OutType,
+          template <typename, typename, typename>
+          typename Epilogue>
 struct sm90_fp8_config_default {
   // M in (128, inf)
   static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
@@ -27,13 +30,19 @@ struct sm90_fp8_config_default {
   using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
   using TileShape = Shape<_128, _128, _128>;
   using ClusterShape = Shape<_2, _1, _1>;
-  using Cutlass3xGemm =
-      cutlass_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
-                      KernelSchedule, EpilogueSchedule>;
+  using Cutlass3xGemm = cutlass_3x_gemm<InType,
+                                        OutType,
+                                        Epilogue,
+                                        TileShape,
+                                        ClusterShape,
+                                        KernelSchedule,
+                                        EpilogueSchedule>;
 };
 
-template <typename InType, typename OutType,
-          template <typename, typename, typename> typename Epilogue>
+template <typename InType,
+          typename OutType,
+          template <typename, typename, typename>
+          typename Epilogue>
 struct sm90_fp8_config_M128 {
   // M in (64, 128]
   static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
@@ -42,13 +51,19 @@ struct sm90_fp8_config_M128 {
   using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
   using TileShape = Shape<_64, _128, _128>;
   using ClusterShape = Shape<_2, _1, _1>;
-  using Cutlass3xGemm =
-      cutlass_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
-                      KernelSchedule, EpilogueSchedule>;
+  using Cutlass3xGemm = cutlass_3x_gemm<InType,
+                                        OutType,
+                                        Epilogue,
+                                        TileShape,
+                                        ClusterShape,
+                                        KernelSchedule,
+                                        EpilogueSchedule>;
 };
 
-template <typename InType, typename OutType,
-          template <typename, typename, typename> typename Epilogue>
+template <typename InType,
+          typename OutType,
+          template <typename, typename, typename>
+          typename Epilogue>
 struct sm90_fp8_config_M64 {
   // M in [1, 64]
   static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
@@ -58,13 +73,19 @@ struct sm90_fp8_config_M64 {
   using TileShape = Shape<_64, _64, _128>;
   using ClusterShape = Shape<_1, _8, _1>;
 
-  using Cutlass3xGemm =
-      cutlass_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
-                      KernelSchedule, EpilogueSchedule>;
+  using Cutlass3xGemm = cutlass_3x_gemm<InType,
+                                        OutType,
+                                        Epilogue,
+                                        TileShape,
+                                        ClusterShape,
+                                        KernelSchedule,
+                                        EpilogueSchedule>;
 };
 
-template <typename InType, typename OutType,
-          template <typename, typename, typename> typename Epilogue,
+template <typename InType,
+          typename OutType,
+          template <typename, typename, typename>
+          typename Epilogue,
           typename... EpilogueArgs>
 inline void cutlass_gemm_sm90_fp8_dispatch(paddle::Tensor &out,
                                            paddle::Tensor const &a,
@@ -75,8 +96,8 @@ inline void cutlass_gemm_sm90_fp8_dispatch(paddle::Tensor &out,
   PD_CHECK(b.dtype() == phi::DataType::FLOAT8_E4M3FN);
 
   using Cutlass3xGemmDefault =
-      typename sm90_fp8_config_default<InType, OutType,
-                                       Epilogue>::Cutlass3xGemm;
+      typename sm90_fp8_config_default<InType, OutType, Epilogue>::
+          Cutlass3xGemm;
   using Cutlass3xGemmM64 =
       typename sm90_fp8_config_M64<InType, OutType, Epilogue>::Cutlass3xGemm;
   using Cutlass3xGemmM128 =
@@ -84,7 +105,7 @@ inline void cutlass_gemm_sm90_fp8_dispatch(paddle::Tensor &out,
 
   uint32_t const m = a.dims()[0];
   uint32_t const mp2 =
-      std::max(static_cast<uint32_t>(64), next_pow_2(m)); // next power of 2
+      std::max(static_cast<uint32_t>(64), next_pow_2(m));  // next power of 2
 
   if (mp2 <= 64) {
     // m in [1, 64]
@@ -112,14 +133,16 @@ void cutlass_scaled_mm_sm90_fp8_epilogue(paddle::Tensor &out,
 
   if (out.dtype() == paddle::DataType::BFLOAT16) {
     return cutlass_gemm_sm90_fp8_dispatch<cutlass::float_e4m3_t,
-                                          cutlass::bfloat16_t, Epilogue>(
+                                          cutlass::bfloat16_t,
+                                          Epilogue>(
         out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
   } else {
     PD_CHECK(out.dtype() == paddle::DataType::FLOAT16);
     return cutlass_gemm_sm90_fp8_dispatch<cutlass::float_e4m3_t,
-                                          cutlass::half_t, Epilogue>(
+                                          cutlass::half_t,
+                                          Epilogue>(
         out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
   }
 }
 
-} // namespace fastdeploy
+}  // namespace fastdeploy

custom_ops/gpu_ops/cutlass_kernels/w8a8/c3x/scaled_mm_sm90_int8.cu

@@ -1,4 +1,5 @@
-// adapted from: https://github.com/vllm-project/vllm/blob/118ff921118cc81061a2af865a1e13840ceb6792/csrc/quantization/cutlass_w8a8/c3x/scaled_mm_sm90_int8.cu
+// adapted from:
+// https://github.com/vllm-project/vllm/blob/118ff921118cc81061a2af865a1e13840ceb6792/csrc/quantization/cutlass_w8a8/c3x/scaled_mm_sm90_int8.cu
 
 // clang-format will break include orders
 // clang-format off
@@ -9,7 +10,8 @@
 
 namespace fastdeploy {
 
-void cutlass_scaled_mm_sm90_int8(paddle::Tensor &out, paddle::Tensor const &a,
+void cutlass_scaled_mm_sm90_int8(paddle::Tensor &out,
+                                 paddle::Tensor const &a,
                                  paddle::Tensor const &b,
                                  paddle::Tensor const &a_scales,
                                  paddle::Tensor const &b_scales,
@@ -17,7 +19,8 @@ void cutlass_scaled_mm_sm90_int8(paddle::Tensor &out, paddle::Tensor const &a,
   PD_CHECK(a_scales.is_contiguous() && b_scales.is_contiguous());
   if (bias) {
     PD_CHECK(bias->dtype() == out.dtype(),
-             "currently bias dtype must match output dtype ", out.dtype());
+             "currently bias dtype must match output dtype ",
+             out.dtype());
     return cutlass_scaled_mm_sm90_int8_epilogue<c3x::ScaledEpilogueBias>(
         out, a, b, a_scales, b_scales, *bias);
   } else {
@@ -26,4 +29,4 @@ void cutlass_scaled_mm_sm90_int8(paddle::Tensor &out, paddle::Tensor const &a,
   }
 }
 
-} // namespace fastdeploy
+}  // namespace fastdeploy

custom_ops/gpu_ops/cutlass_kernels/w8a8/c3x/scaled_mm_sm90_int8_dispatch.cuh

@@ -1,4 +1,5 @@
-// adapted from: https://github.com/vllm-project/vllm/blob/118ff921118cc81061a2af865a1e13840ceb6792/csrc/quantization/cutlass_w8a8/c3x/scaled_mm_sm90_int8_dispatch.cuh
+// adapted from:
+// https://github.com/vllm-project/vllm/blob/118ff921118cc81061a2af865a1e13840ceb6792/csrc/quantization/cutlass_w8a8/c3x/scaled_mm_sm90_int8_dispatch.cuh
 
 #pragma once
 
@@ -17,8 +18,10 @@ namespace fastdeploy {
 
 using c3x::cutlass_gemm_caller;
 
-template <typename InType, typename OutType,
-          template <typename, typename, typename> typename Epilogue>
+template <typename InType,
+          typename OutType,
+          template <typename, typename, typename>
+          typename Epilogue>
 struct sm90_int8_config_default {
   // For M > 128 and any N
   static_assert(std::is_same<InType, int8_t>());
@@ -27,13 +30,19 @@ struct sm90_int8_config_default {
   using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
   using TileShape = Shape<_128, _128, _128>;
   using ClusterShape = Shape<_2, _1, _1>;
-  using Cutlass3xGemm =
-      cutlass_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
-                      KernelSchedule, EpilogueSchedule>;
+  using Cutlass3xGemm = cutlass_3x_gemm<InType,
+                                        OutType,
+                                        Epilogue,
+                                        TileShape,
+                                        ClusterShape,
+                                        KernelSchedule,
+                                        EpilogueSchedule>;
 };
 
-template <typename InType, typename OutType,
-          template <typename, typename, typename> typename Epilogue>
+template <typename InType,
+          typename OutType,
+          template <typename, typename, typename>
+          typename Epilogue>
 struct sm90_int8_config_M128 {
   // For M in (64, 128] and any N
   static_assert(std::is_same<InType, int8_t>());
@@ -42,13 +51,19 @@ struct sm90_int8_config_M128 {
   using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
   using TileShape = Shape<_64, _128, _128>;
   using ClusterShape = Shape<_2, _1, _1>;
-  using Cutlass3xGemm =
-      cutlass_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
-                      KernelSchedule, EpilogueSchedule>;
+  using Cutlass3xGemm = cutlass_3x_gemm<InType,
+                                        OutType,
+                                        Epilogue,
+                                        TileShape,
+                                        ClusterShape,
+                                        KernelSchedule,
+                                        EpilogueSchedule>;
 };
 
-template <typename InType, typename OutType,
-          template <typename, typename, typename> typename Epilogue>
+template <typename InType,
+          typename OutType,
+          template <typename, typename, typename>
+          typename Epilogue>
 struct sm90_int8_config_M64 {
   // For M in (32, 64] and any N
   static_assert(std::is_same<InType, int8_t>());
@@ -56,13 +71,19 @@ struct sm90_int8_config_M64 {
   using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
   using TileShape = Shape<_64, _64, _256>;
   using ClusterShape = Shape<_1, _1, _1>;
-  using Cutlass3xGemm =
-      cutlass_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
-                      KernelSchedule, EpilogueSchedule>;
+  using Cutlass3xGemm = cutlass_3x_gemm<InType,
+                                        OutType,
+                                        Epilogue,
+                                        TileShape,
+                                        ClusterShape,
+                                        KernelSchedule,
+                                        EpilogueSchedule>;
 };
 
-template <typename InType, typename OutType,
-          template <typename, typename, typename> typename Epilogue>
+template <typename InType,
+          typename OutType,
+          template <typename, typename, typename>
+          typename Epilogue>
 struct sm90_int8_config_M32_NBig {
   // For M in [1, 32] and N >= 8192
   static_assert(std::is_same<InType, int8_t>());
@@ -70,13 +91,19 @@ struct sm90_int8_config_M32_NBig {
   using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
   using TileShape = Shape<_64, _128, _256>;
   using ClusterShape = Shape<_1, _4, _1>;
-  using Cutlass3xGemm =
-      cutlass_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
-                      KernelSchedule, EpilogueSchedule>;
+  using Cutlass3xGemm = cutlass_3x_gemm<InType,
+                                        OutType,
+                                        Epilogue,
+                                        TileShape,
+                                        ClusterShape,
+                                        KernelSchedule,
+                                        EpilogueSchedule>;
 };
 
-template <typename InType, typename OutType,
-          template <typename, typename, typename> typename Epilogue>
+template <typename InType,
+          typename OutType,
+          template <typename, typename, typename>
+          typename Epilogue>
 struct sm90_int8_config_M32_NSmall {
   // For M in [1, 32] and N < 8192
   static_assert(std::is_same<InType, int8_t>());
@@ -84,13 +111,19 @@ struct sm90_int8_config_M32_NSmall {
   using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
   using TileShape = Shape<_64, _64, _256>;
   using ClusterShape = Shape<_1, _8, _1>;
-  using Cutlass3xGemm =
-      cutlass_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
-                      KernelSchedule, EpilogueSchedule>;
+  using Cutlass3xGemm = cutlass_3x_gemm<InType,
+                                        OutType,
+                                        Epilogue,
+                                        TileShape,
+                                        ClusterShape,
+                                        KernelSchedule,
+                                        EpilogueSchedule>;
 };
 
-template <typename InType, typename OutType,
-          template <typename, typename, typename> typename Epilogue,
+template <typename InType,
+          typename OutType,
+          template <typename, typename, typename>
+          typename Epilogue,
           typename... EpilogueArgs>
 inline void cutlass_gemm_sm90_int8_dispatch(paddle::Tensor &out,
                                             paddle::Tensor const &a,
@@ -101,25 +134,25 @@ inline void cutlass_gemm_sm90_int8_dispatch(paddle::Tensor &out,
   PD_CHECK(b.dtype() == paddle::DataType::INT8);
 
   using Cutlass3xGemmDefault =
-      typename sm90_int8_config_default<InType, OutType,
-                                        Epilogue>::Cutlass3xGemm;
+      typename sm90_int8_config_default<InType, OutType, Epilogue>::
+          Cutlass3xGemm;
   using Cutlass3xGemmM128 =
       typename sm90_int8_config_M128<InType, OutType, Epilogue>::Cutlass3xGemm;
   using Cutlass3xGemmM64 =
       typename sm90_int8_config_M64<InType, OutType, Epilogue>::Cutlass3xGemm;
   using Cutlass3xGemmM32NBig =
-      typename sm90_int8_config_M32_NBig<InType, OutType,
-                                         Epilogue>::Cutlass3xGemm;
+      typename sm90_int8_config_M32_NBig<InType, OutType, Epilogue>::
+          Cutlass3xGemm;
   using Cutlass3xGemmM32NSmall =
-      typename sm90_int8_config_M32_NSmall<InType, OutType,
-                                           Epilogue>::Cutlass3xGemm;
+      typename sm90_int8_config_M32_NSmall<InType, OutType, Epilogue>::
+          Cutlass3xGemm;
 
   uint32_t const n = out.dims()[1];
   bool const is_small_n = n < 8192;
 
   uint32_t const m = a.dims()[0];
   uint32_t const mp2 =
-      std::max(static_cast<uint32_t>(32), next_pow_2(m)); // next power of 2
+      std::max(static_cast<uint32_t>(32), next_pow_2(m));  // next power of 2
 
   if (mp2 <= 32) {
     // m in [1, 32]
@@ -155,7 +188,8 @@ void cutlass_scaled_mm_sm90_int8_epilogue(paddle::Tensor &out,
   PD_CHECK(b.dtype() == paddle::DataType::INT8);
 
   if (out.dtype() == paddle::DataType::BFLOAT16) {
-    return cutlass_gemm_sm90_int8_dispatch<int8_t, cutlass::bfloat16_t,
+    return cutlass_gemm_sm90_int8_dispatch<int8_t,
+                                           cutlass::bfloat16_t,
                                            Epilogue>(
         out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
   } else {
@@ -165,4 +199,4 @@ void cutlass_scaled_mm_sm90_int8_epilogue(paddle::Tensor &out,
   }
 }
 
-} // namespace fastdeploy
+}  // namespace fastdeploy