apps/FastDeploy
1
0
Fork 0
mirror of https://github.com/PaddlePaddle/FastDeploy.git synced 2026-04-23 00:17:25 +08:00
Code Issues Actions 21 Packages Projects Releases Wiki Activity
Files
007c3ad0faf067fa1cf9b6d1d44e30033fbcb57a
FastDeploy/get_started/ernie-4.5-vl-thinking/index.html
T

2964 lines
55 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!doctype html>
<html lang="en" class="no-js">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<link rel="alternate" href="./" hreflang="en">
<link rel="alternate" href="../../zh/get_started/ernie-4.5-vl-thinking/" hreflang="zh">
<link rel="icon" href="../../assets/images/favicon.ico">
<meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.7.6">
<title>Ernie 4.5 vl thinking - FastDeploy: Large Language Model Deployment</title>
<link rel="stylesheet" href="../../assets/stylesheets/main.484c7ddc.min.css">
<link rel="stylesheet" href="../../assets/stylesheets/palette.ab4e12ef.min.css">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
<style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
<script>__md_scope=new URL("../..",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
</head>
<body dir="ltr" data-md-color-scheme="default" data-md-color-primary="indigo" data-md-color-accent="indigo">
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
<label class="md-overlay" for="__drawer"></label>
<div data-md-component="skip">
<a href="#deploy-ernie-45-vl-28b-a3b-thinking-multimodal-thinking-model" class="md-skip">
Skip to content
</a>
</div>
<div data-md-component="announce">
</div>
<header class="md-header md-header--shadow" data-md-component="header">
<nav class="md-header__inner md-grid" aria-label="Header">
<a href="../.." title="FastDeploy: Large Language Model Deployment" class="md-header__button md-logo" aria-label="FastDeploy: Large Language Model Deployment" data-md-component="logo">
<img src="../../assets/images/logo.jpg" alt="logo">
</a>
<label class="md-header__button md-icon" for="__drawer">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
</label>
<div class="md-header__title" data-md-component="header-title">
<div class="md-header__ellipsis">
<div class="md-header__topic">
<span class="md-ellipsis">
FastDeploy: Large Language Model Deployment
</span>
</div>
<div class="md-header__topic" data-md-component="header-topic">
<span class="md-ellipsis">
Ernie 4.5 vl thinking
</span>
</div>
</div>
</div>
<form class="md-header__option" data-md-component="palette">
<input class="md-option" data-md-color-media="(prefers-color-scheme: light)" data-md-color-scheme="default" data-md-color-primary="indigo" data-md-color-accent="indigo" aria-label="Switch to dark mode" type="radio" name="__palette" id="__palette_0">
<label class="md-header__button md-icon" title="Switch to dark mode" for="__palette_1" hidden>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a4 4 0 0 0-4 4 4 4 0 0 0 4 4 4 4 0 0 0 4-4 4 4 0 0 0-4-4m0 10a6 6 0 0 1-6-6 6 6 0 0 1 6-6 6 6 0 0 1 6 6 6 6 0 0 1-6 6m8-9.31V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12z"/></svg>
</label>
<input class="md-option" data-md-color-media="(prefers-color-scheme: dark)" data-md-color-scheme="slate" data-md-color-primary="black" data-md-color-accent="indigo" aria-label="Switch to system preference" type="radio" name="__palette" id="__palette_1">
<label class="md-header__button md-icon" title="Switch to system preference" for="__palette_0" hidden>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 18c-.89 0-1.74-.2-2.5-.55C11.56 16.5 13 14.42 13 12s-1.44-4.5-3.5-5.45C10.26 6.2 11.11 6 12 6a6 6 0 0 1 6 6 6 6 0 0 1-6 6m8-9.31V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12z"/></svg>
</label>
</form>
<script>var palette=__md_get("__palette");if(palette&&palette.color){if("(prefers-color-scheme)"===palette.color.media){var media=matchMedia("(prefers-color-scheme: light)"),input=document.querySelector(media.matches?"[data-md-color-media='(prefers-color-scheme: light)']":"[data-md-color-media='(prefers-color-scheme: dark)']");palette.color.media=input.getAttribute("data-md-color-media"),palette.color.scheme=input.getAttribute("data-md-color-scheme"),palette.color.primary=input.getAttribute("data-md-color-primary"),palette.color.accent=input.getAttribute("data-md-color-accent")}for(var[key,value]of Object.entries(palette.color))document.body.setAttribute("data-md-color-"+key,value)}</script>
<div class="md-header__option">
<div class="md-select">
<button class="md-header__button md-icon" aria-label="Select language">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="m12.87 15.07-2.54-2.51.03-.03A17.5 17.5 0 0 0 14.07 6H17V4h-7V2H8v2H1v2h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2zm-2.62 7 1.62-4.33L19.12 17z"/></svg>
</button>
<div class="md-select__inner">
<ul class="md-select__list">
<li class="md-select__item">
<a href="./" hreflang="en" class="md-select__link">
English
</a>
</li>
<li class="md-select__item">
<a href="../../zh/get_started/ernie-4.5-vl-thinking/" hreflang="zh" class="md-select__link">
简体中文
</a>
</li>
</ul>
</div>
</div>
</div>
<label class="md-header__button md-icon" for="__search">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
</label>
<div class="md-search" data-md-component="search" role="dialog">
<label class="md-search__overlay" for="__search"></label>
<div class="md-search__inner" role="search">
<form class="md-search__form" name="search">
<input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
<label class="md-search__icon md-icon" for="__search">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg>
</label>
<nav class="md-search__options" aria-label="Search">
<button type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z"/></svg>
</button>
</nav>
</form>
<div class="md-search__output">
<div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
<div class="md-search-result" data-md-component="search-result">
<div class="md-search-result__meta">
Initializing search
</div>
<ol class="md-search-result__list" role="presentation"></ol>
</div>
</div>
</div>
</div>
</div>
<div class="md-header__source">
<a href="https://github.com/PaddlePaddle/FastDeploy" title="Go to repository" class="md-source" data-md-component="source">
<div class="md-source__icon md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M439.6 236.1 244 40.5c-5.4-5.5-12.8-8.5-20.4-8.5s-15 3-20.4 8.4L162.5 81l51.5 51.5c27.1-9.1 52.7 16.8 43.4 43.7l49.7 49.7c34.2-11.8 61.2 31 35.5 56.7-26.5 26.5-70.2-2.9-56-37.3L240.3 199v121.9c25.3 12.5 22.3 41.8 9.1 55-6.4 6.4-15.2 10.1-24.3 10.1s-17.8-3.6-24.3-10.1c-17.6-17.6-11.1-46.9 11.2-56v-123c-20.8-8.5-24.6-30.7-18.6-45L142.6 101 8.5 235.1C3 240.6 0 247.9 0 255.5s3 15 8.5 20.4l195.6 195.7c5.4 5.4 12.7 8.4 20.4 8.4s15-3 20.4-8.4l194.7-194.7c5.4-5.4 8.4-12.8 8.4-20.4s-3-15-8.4-20.4"/></svg>
</div>
<div class="md-source__repository">
FastDeploy
</div>
</a>
</div>
</nav>
</header>
<div class="md-container" data-md-component="container">
<main class="md-main" data-md-component="main">
<div class="md-main__inner md-grid">
<div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--primary" aria-label="Navigation" data-md-level="0">
<label class="md-nav__title" for="__drawer">
<a href="../.." title="FastDeploy: Large Language Model Deployment" class="md-nav__button md-logo" aria-label="FastDeploy: Large Language Model Deployment" data-md-component="logo">
<img src="../../assets/images/logo.jpg" alt="logo">
</a>
FastDeploy: Large Language Model Deployment
</label>
<div class="md-nav__source">
<a href="https://github.com/PaddlePaddle/FastDeploy" title="Go to repository" class="md-source" data-md-component="source">
<div class="md-source__icon md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M439.6 236.1 244 40.5c-5.4-5.5-12.8-8.5-20.4-8.5s-15 3-20.4 8.4L162.5 81l51.5 51.5c27.1-9.1 52.7 16.8 43.4 43.7l49.7 49.7c34.2-11.8 61.2 31 35.5 56.7-26.5 26.5-70.2-2.9-56-37.3L240.3 199v121.9c25.3 12.5 22.3 41.8 9.1 55-6.4 6.4-15.2 10.1-24.3 10.1s-17.8-3.6-24.3-10.1c-17.6-17.6-11.1-46.9 11.2-56v-123c-20.8-8.5-24.6-30.7-18.6-45L142.6 101 8.5 235.1C3 240.6 0 247.9 0 255.5s3 15 8.5 20.4l195.6 195.7c5.4 5.4 12.7 8.4 20.4 8.4s15-3 20.4-8.4l194.7-194.7c5.4-5.4 8.4-12.8 8.4-20.4s-3-15-8.4-20.4"/></svg>
</div>
<div class="md-source__repository">
FastDeploy
</div>
</a>
</div>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../.." class="md-nav__link">
<span class="md-ellipsis">
FastDeploy
</span>
</a>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2" >
<label class="md-nav__link" for="__nav_2" id="__nav_2_label" tabindex="0">
<span class="md-ellipsis">
Quick Start
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_2_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_2">
<span class="md-nav__icon md-icon"></span>
Quick Start
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2_1" >
<label class="md-nav__link" for="__nav_2_1" id="__nav_2_1_label" tabindex="0">
<span class="md-ellipsis">
Installation
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_1_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_2_1">
<span class="md-nav__icon md-icon"></span>
Installation
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../installation/nvidia_gpu/" class="md-nav__link">
<span class="md-ellipsis">
Nvidia GPU
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../installation/kunlunxin_xpu/" class="md-nav__link">
<span class="md-ellipsis">
KunlunXin XPU
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../installation/intel_gaudi/" class="md-nav__link">
<span class="md-ellipsis">
Intel Gaudi
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../installation/hygon_dcu/" class="md-nav__link">
<span class="md-ellipsis">
HYGON DCU
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../installation/Enflame_gcu/" class="md-nav__link">
<span class="md-ellipsis">
Enflame S60
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../installation/iluvatar_gpu/" class="md-nav__link">
<span class="md-ellipsis">
Iluvatar CoreX
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../installation/metax_gpu/" class="md-nav__link">
<span class="md-ellipsis">
Metax C550
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="../quick_start/" class="md-nav__link">
<span class="md-ellipsis">
Quick Deployment For ERNIE-4.5-0.3B
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../quick_start_vl/" class="md-nav__link">
<span class="md-ellipsis">
Quick Deployment for ERNIE-4.5-VL-28B-A3B
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../ernie-4.5/" class="md-nav__link">
<span class="md-ellipsis">
ERNIE-4.5-300B-A47B
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../ernie-4.5-vl/" class="md-nav__link">
<span class="md-ellipsis">
ERNIE-4.5-VL-424B-A47B
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../quick_start_qwen/" class="md-nav__link">
<span class="md-ellipsis">
Quick Deployment For QWEN
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../quick_start_qwen25_vl/" class="md-nav__link">
<span class="md-ellipsis">
Quick Deployment For QWEN2.5-VL
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3" >
<label class="md-nav__link" for="__nav_3" id="__nav_3_label" tabindex="0">
<span class="md-ellipsis">
Online Serving
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_3_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_3">
<span class="md-nav__icon md-icon"></span>
Online Serving
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../online_serving/" class="md-nav__link">
<span class="md-ellipsis">
OpenAI-Compatible API Server
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../online_serving/metrics/" class="md-nav__link">
<span class="md-ellipsis">
Monitor Metrics
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../online_serving/scheduler/" class="md-nav__link">
<span class="md-ellipsis">
Scheduler
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../online_serving/graceful_shutdown_service/" class="md-nav__link">
<span class="md-ellipsis">
Graceful Shutdown
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../online_serving/router/" class="md-nav__link">
<span class="md-ellipsis">
Load-Balancing Scheduling Router
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="../../offline_inference/" class="md-nav__link">
<span class="md-ellipsis">
Offline Inference
</span>
</a>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_5" >
<label class="md-nav__link" for="__nav_5" id="__nav_5_label" tabindex="0">
<span class="md-ellipsis">
Best Practices
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_5_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_5">
<span class="md-nav__icon md-icon"></span>
Best Practices
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../best_practices/ERNIE-4.5-0.3B-Paddle/" class="md-nav__link">
<span class="md-ellipsis">
ERNIE-4.5-0.3B
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../best_practices/ERNIE-4.5-21B-A3B-Paddle/" class="md-nav__link">
<span class="md-ellipsis">
ERNIE-4.5-21B-A3B
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../best_practices/ERNIE-4.5-300B-A47B-Paddle/" class="md-nav__link">
<span class="md-ellipsis">
ERNIE-4.5-300B-A47B
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../best_practices/ERNIE-4.5-21B-A3B-Thinking/" class="md-nav__link">
<span class="md-ellipsis">
ERNIE-4.5-21B-A3B-Thinking
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../best_practices/ERNIE-4.5-VL-28B-A3B-Paddle/" class="md-nav__link">
<span class="md-ellipsis">
ERNIE-4.5-VL-28B-A3B
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../best_practices/ERNIE-4.5-VL-424B-A47B-Paddle/" class="md-nav__link">
<span class="md-ellipsis">
ERNIE-4.5-VL-424B-A47B
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../best_practices/PaddleOCR-VL-0.9B/" class="md-nav__link">
<span class="md-ellipsis">
PaddleOCR-VL-0.9B
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../best_practices/FAQ/" class="md-nav__link">
<span class="md-ellipsis">
FAQ
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6" >
<label class="md-nav__link" for="__nav_6" id="__nav_6_label" tabindex="0">
<span class="md-ellipsis">
Quantization
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_6_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_6">
<span class="md-nav__icon md-icon"></span>
Quantization
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../quantization/" class="md-nav__link">
<span class="md-ellipsis">
Overview
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../quantization/online_quantization/" class="md-nav__link">
<span class="md-ellipsis">
Online Quantization
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../quantization/wint2/" class="md-nav__link">
<span class="md-ellipsis">
WINT2 Quantization
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_7" >
<label class="md-nav__link" for="__nav_7" id="__nav_7_label" tabindex="0">
<span class="md-ellipsis">
Features
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_7_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_7">
<span class="md-nav__icon md-icon"></span>
Features
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../features/prefix_caching/" class="md-nav__link">
<span class="md-ellipsis">
Prefix Caching
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../features/disaggregated/" class="md-nav__link">
<span class="md-ellipsis">
Disaggregation
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../features/chunked_prefill/" class="md-nav__link">
<span class="md-ellipsis">
Chunked Prefill
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../features/load_balance/" class="md-nav__link">
<span class="md-ellipsis">
Load Balance
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../features/speculative_decoding/" class="md-nav__link">
<span class="md-ellipsis">
Speculative Decoding
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../features/structured_outputs/" class="md-nav__link">
<span class="md-ellipsis">
Structured Outputs
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../features/reasoning_output/" class="md-nav__link">
<span class="md-ellipsis">
Reasoning Output
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../features/thinking_budget/" class="md-nav__link">
<span class="md-ellipsis">
Thinking Budget
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../features/early_stop/" class="md-nav__link">
<span class="md-ellipsis">
Early Stop
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../features/plugins/" class="md-nav__link">
<span class="md-ellipsis">
Plugins
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../features/paddleformers_backend/" class="md-nav__link">
<span class="md-ellipsis">
PaddleFormers Backend
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../features/sampling/" class="md-nav__link">
<span class="md-ellipsis">
Sampling
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../features/multi-node_deployment/" class="md-nav__link">
<span class="md-ellipsis">
MultiNode Deployment
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../features/graph_optimization/" class="md-nav__link">
<span class="md-ellipsis">
Graph Optimization
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../features/data_parallel_service/" class="md-nav__link">
<span class="md-ellipsis">
Data Parallelism
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../features/plas_attention/" class="md-nav__link">
<span class="md-ellipsis">
PLAS
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="../../supported_models/" class="md-nav__link">
<span class="md-ellipsis">
Supported Models
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../benchmark/" class="md-nav__link">
<span class="md-ellipsis">
Benchmark
</span>
</a>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
<label class="md-nav__link" for="__nav_10" id="__nav_10_label" tabindex="0">
<span class="md-ellipsis">
Usage
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_10">
<span class="md-nav__icon md-icon"></span>
Usage
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../usage/log/" class="md-nav__link">
<span class="md-ellipsis">
Log Description
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../usage/code_overview/" class="md-nav__link">
<span class="md-ellipsis">
Code Overview
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../usage/environment_variables/" class="md-nav__link">
<span class="md-ellipsis">
Environment Variables
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
<label class="md-nav__link" for="__nav_11" id="__nav_11_label" tabindex="0">
<span class="md-ellipsis">
CLI
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_11">
<span class="md-nav__icon md-icon"></span>
CLI
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../cli/" class="md-nav__link">
<span class="md-ellipsis">
Overview
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../cli/chat/" class="md-nav__link">
<span class="md-ellipsis">
Chat
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../cli/complete/" class="md-nav__link">
<span class="md-ellipsis">
Complete
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../cli/serve/" class="md-nav__link">
<span class="md-ellipsis">
Server
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../cli/collect-env/" class="md-nav__link">
<span class="md-ellipsis">
Collect Env
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../cli/bench/" class="md-nav__link">
<span class="md-ellipsis">
Bench
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../cli/run-batch/" class="md-nav__link">
<span class="md-ellipsis">
Run Batch
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../cli/tokenizer/" class="md-nav__link">
<span class="md-ellipsis">
Tokenizer
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_12" >
<label class="md-nav__link" for="__nav_12" id="__nav_12_label" tabindex="0">
<span class="md-ellipsis">
Observability
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_12_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_12">
<span class="md-nav__icon md-icon"></span>
Observability
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../observability/trace/" class="md-nav__link">
<span class="md-ellipsis">
Trace
</span>
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
<label class="md-nav__title" for="__toc">
<span class="md-nav__icon md-icon"></span>
Table of contents
</label>
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
<li class="md-nav__item">
<a href="#prepare-the-model" class="md-nav__link">
<span class="md-ellipsis">
Prepare the Model
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#launch-the-service" class="md-nav__link">
<span class="md-ellipsis">
Launch the Service
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#request-the-service" class="md-nav__link">
<span class="md-ellipsis">
Request the Service
</span>
</a>
<nav class="md-nav" aria-label="Request the Service">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#health-check" class="md-nav__link">
<span class="md-ellipsis">
Health Check
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#curl-request" class="md-nav__link">
<span class="md-ellipsis">
cURL Request
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#python-client-openai-compatible-api" class="md-nav__link">
<span class="md-ellipsis">
Python Client (OpenAI-compatible API)
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#model-output" class="md-nav__link">
<span class="md-ellipsis">
Model Output
</span>
</a>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-content" data-md-component="content">
<article class="md-content__inner md-typeset">
<p><a href="../zh/get_started/ernie-4.5-vl-thinking.md">简体中文</a></p>
<h1 id="deploy-ernie-45-vl-28b-a3b-thinking-multimodal-thinking-model">Deploy ERNIE-4.5-VL-28B-A3B-Thinking Multimodal Thinking Model</h1>
<p>This document explains how to deploy the ERNIE-4.5-VL multimodal model, supporting user interaction via multimodal data and tool call (including for multimodal data). Ensure your hardware meets the requirements before deployment.</p>
<ul>
<li>GPU Driver &gt;= 535</li>
<li>CUDA &gt;= 12.3</li>
<li>CUDNN &gt;= 9.5</li>
<li>Linux X86_64</li>
<li>Python &gt;= 3.10</li>
<li>80G A/H 1 GPUs</li>
</ul>
<p>Refer to the <a href="../installation/">Installation Guide</a> for FastDeploy setup.</p>
<h2 id="prepare-the-model">Prepare the Model</h2>
<p>Specify <code>--model baidu/ERNIE-4.5-VL-28B-A3B-Thinking</code> during deployment to automatically download the model from AIStudio with resumable downloads. You can also manually download the model from other sources. Note that FastDeploy requires Paddle-format models. For more details, see <a href="../../supported_models/">Supported Models</a>.</p>
<h2 id="launch-the-service">Launch the Service</h2>
<p>Execute the following command to start the service. For parameter configurations, refer to <a href="../../parameters/">Parameter Guide</a>.</p>
<pre><code class="language-shell">python -m fastdeploy.entrypoints.openai.api_server \
--model baidu/ERNIE-4.5-VL-28B-A3B-Thinking \
--max-model-len 131072 \
--max-num-seqs 32 \
--port 8180 \
--quantization wint8 \
--reasoning-parser ernie-45-vl-thinking \
--tool-call-parser ernie-45-vl-thinking \
--mm-processor-kwargs '{&quot;image_max_pixels&quot;: 12845056 }'
</code></pre>
<h2 id="request-the-service">Request the Service</h2>
<p>After launching, the service is ready when the following logs appear:</p>
<pre><code class="language-shell">api_server.py[line:91] Launching metrics service at http://0.0.0.0:8181/metrics
api_server.py[line:94] Launching chat completion service at http://0.0.0.0:8180/v1/chat/completions
api_server.py[line:97] Launching completion service at http://0.0.0.0:8180/v1/completions
INFO: Started server process [13909]
INFO: Waiting for application startup.
INFO: Application startup complete.
INFO: Uvicorn running on http://0.0.0.0:8180 (Press CTRL+C to quit)
</code></pre>
<h3 id="health-check">Health Check</h3>
<p>Verify service status (HTTP 200 indicates success):</p>
<pre><code class="language-shell">curl -i http://0.0.0.0:8180/health
</code></pre>
<h3 id="curl-request">cURL Request</h3>
<p>Send requests as follows:</p>
<pre><code class="language-shell">curl -X POST &quot;http://0.0.0.0:8180/v1/chat/completions&quot; \
-H &quot;Content-Type: application/json&quot; \
-d '{
&quot;messages&quot;: [
{&quot;role&quot;: &quot;user&quot;, &quot;content&quot;: &quot;Rewrite Li Bai's 'Quiet Night Thoughts' as a modern poem&quot;}
]
}'
</code></pre>
<p>\
For image inputs:</p>
<pre><code class="language-shell">curl -X POST &quot;http://0.0.0.0:8180/v1/chat/completions&quot; \
-H &quot;Content-Type: application/json&quot; \
-d '{
&quot;messages&quot;: [
{&quot;role&quot;: &quot;user&quot;, &quot;content&quot;: [
{&quot;type&quot;:&quot;image_url&quot;, &quot;image_url&quot;: {&quot;url&quot;:&quot;https://paddlenlp.bj.bcebos.com/datasets/paddlemix/demo_images/example2.jpg&quot;}},
{&quot;type&quot;:&quot;text&quot;, &quot;text&quot;:&quot;From which era does the artifact in the image originate?&quot;}
]}
]
}'
</code></pre>
<p>Image can also be provided through base64-encoded string:</p>
<pre><code class="language-shell">{&quot;type&quot;:&quot;image_url&quot;, &quot;image_url&quot;: {&quot;url&quot;:&quot;data:image/jpg;base64,this/is/an/example&quot;}
</code></pre>
<p>or absolute path to local file:</p>
<pre><code class="language-shell">{&quot;type&quot;:&quot;image_url&quot;, &quot;image_url&quot;: {&quot;url&quot;:&quot;file:///this/is/an/example&quot;}
</code></pre>
<p>\
For video inputs:</p>
<pre><code class="language-shell">curl -X POST &quot;http://0.0.0.0:8180/v1/chat/completions&quot; \
-H &quot;Content-Type: application/json&quot; \
-d '{
&quot;messages&quot;: [
{&quot;role&quot;: &quot;user&quot;, &quot;content&quot;: [
{&quot;type&quot;:&quot;video_url&quot;, &quot;video_url&quot;: {&quot;url&quot;:&quot;https://bj.bcebos.com/v1/paddlenlp/datasets/paddlemix/demo_video/example_video.mp4&quot;}},
{&quot;type&quot;:&quot;text&quot;, &quot;text&quot;:&quot;How many apples are in the scene?&quot;}
]}
]
}'
</code></pre>
<p>Video can also be provided through base64-encoded string:</p>
<pre><code class="language-shell">{&quot;type&quot;:&quot;video_url&quot;, &quot;video_url&quot;: {&quot;url&quot;:&quot;data:video/mp4;base64,this/is/an/example&quot;}
</code></pre>
<p>or absolute path to local file:</p>
<pre><code class="language-shell">{&quot;type&quot;:&quot;video_url&quot;, &quot;video_url&quot;: {&quot;url&quot;:&quot;file:///this/is/an/example&quot;}
</code></pre>
<p>\
Input includes tool calls, send requests with the command below:</p>
<pre><code class="language-shell">curl -X POST &quot;http://0.0.0.0:8180/v1/chat/completions&quot; \
-H &quot;Content-Type: application/json&quot; \
-d $'{
&quot;tools&quot;: [
{
&quot;type&quot;: &quot;function&quot;,
&quot;function&quot;: {
&quot;name&quot;: &quot;image_zoom_in_tool&quot;,
&quot;description&quot;: &quot;Zoom in on a specific region of an image by cropping it based on a bounding box (bbox) and an optional object label.&quot;,
&quot;parameters&quot;: {
&quot;type&quot;: &quot;object&quot;,
&quot;properties&quot;: {
&quot;bbox_2d&quot;: {
&quot;type&quot;: &quot;array&quot;,
&quot;items&quot;: {
&quot;type&quot;: &quot;number&quot;
},
&quot;minItems&quot;: 4,
&quot;maxItems&quot;: 4,
&quot;description&quot;: &quot;The bounding box of the region to zoom in, as [x1, y1, x2, y2], where (x1, y1) is the top-left corner and (x2, y2) is the bottom-right corner, and the values of x1, y1, x2, y2 are all normalized to the range 0–1000 based on the original image dimensions.&quot;
},
&quot;label&quot;: {
&quot;type&quot;: &quot;string&quot;,
&quot;description&quot;: &quot;The name or label of the object in the specified bounding box (optional).&quot;
}
},
&quot;required&quot;: [
&quot;bbox_2d&quot;
]
},
&quot;strict&quot;: false
}
}
],
&quot;messages&quot;: [
{
&quot;role&quot;: &quot;user&quot;,
&quot;content&quot;: [
{
&quot;type&quot;: &quot;text&quot;,
&quot;text&quot;: &quot;Is the old lady on the left side of the empty table behind older couple?&quot;
}
]
}
],
&quot;stream&quot;: false
}'
</code></pre>
<p>For multi-round requests with tool results in history context, use the command below:</p>
<pre><code class="language-shell">curl -X POST &quot;http://0.0.0.0:8180/v1/chat/completions&quot; \
-H &quot;Content-Type: application/json&quot; \
-d $'{
&quot;messages&quot;: [
{
&quot;role&quot;: &quot;user&quot;,
&quot;content&quot;: [
{
&quot;type&quot;: &quot;text&quot;,
&quot;text&quot;: &quot;Get the current weather in Beijing&quot;
}
]
},
{
&quot;role&quot;: &quot;assistant&quot;,
&quot;tool_calls&quot;: [
{
&quot;id&quot;: &quot;call_1&quot;,
&quot;type&quot;: &quot;function&quot;,
&quot;function&quot;: {
&quot;name&quot;: &quot;get_weather&quot;,
&quot;arguments&quot;: {
&quot;location&quot;: &quot;Beijing&quot;,
&quot;unit&quot;: &quot;c&quot;
}
}
}
],
&quot;content&quot;: &quot;&quot;
},
{
&quot;role&quot;: &quot;tool&quot;,
&quot;content&quot;: [
{
&quot;type&quot;: &quot;text&quot;,
&quot;text&quot;: &quot;location: Beijing，temperature: 23，weather: sunny，unit: c&quot;
}
]
}
],
&quot;tools&quot;: [
{
&quot;type&quot;: &quot;function&quot;,
&quot;function&quot;: {
&quot;name&quot;: &quot;get_weather&quot;,
&quot;description&quot;: &quot;Determine weather in my location&quot;,
&quot;parameters&quot;: {
&quot;type&quot;: &quot;object&quot;,
&quot;properties&quot;: {
&quot;location&quot;: {
&quot;type&quot;: &quot;string&quot;,
&quot;description&quot;: &quot;The city and state e.g. San Francisco, CA&quot;
},
&quot;unit&quot;: {
&quot;type&quot;: &quot;string&quot;,
&quot;enum&quot;: [
&quot;c&quot;,
&quot;f&quot;
]
}
},
&quot;additionalProperties&quot;: false,
&quot;required&quot;: [
&quot;location&quot;,
&quot;unit&quot;
]
},
&quot;strict&quot;: true
}
}
],
&quot;stream&quot;: false
}'
</code></pre>
<h3 id="python-client-openai-compatible-api">Python Client (OpenAI-compatible API)</h3>
<p>FastDeploy's API is OpenAI-compatible. You can also use Python for streaming requests:</p>
<pre><code class="language-python">import openai
host = &quot;0.0.0.0&quot;
port = &quot;8180&quot;
client = openai.Client(base_url=f&quot;http://{host}:{port}/v1&quot;, api_key=&quot;null&quot;)
response = client.chat.completions.create(
model=&quot;null&quot;,
messages=[
{&quot;role&quot;: &quot;user&quot;, &quot;content&quot;: [
{&quot;type&quot;: &quot;image_url&quot;, &quot;image_url&quot;: {&quot;url&quot;: &quot;https://paddlenlp.bj.bcebos.com/datasets/paddlemix/demo_images/example2.jpg&quot;}},
{&quot;type&quot;: &quot;text&quot;, &quot;text&quot;: &quot;From which era does the artifact in the image originate?&quot;},
]},
],
stream=True,
)
for chunk in response:
if chunk.choices[0].delta:
print(chunk.choices[0].delta.content, end='')
print('\n')
</code></pre>
<h2 id="model-output">Model Output</h2>
<p>Example output with reasoning (reasoning content in <code>reasoning_content</code>, response in <code>content</code>, tool_calls in <code>tool_calls</code>):</p>
<p>Example of non-streaming results without tool call:</p>
<pre><code class="language-json">{
&quot;choices&quot;: [
{
&quot;index&quot;: 0,
&quot;message&quot;: {
&quot;role&quot;: &quot;assistant&quot;,
&quot;content&quot;: &quot;The artifact in the image ...&quot;,
&quot;multimodal_content&quot;: null,
&quot;reasoning_content&quot;: &quot;The user asks about ...&quot;,
&quot;tool_calls&quot;: null
},
&quot;finish_reason&quot;: &quot;stop&quot;
}
],
&quot;usage&quot;: {
&quot;prompt_tokens&quot;: 1290,
&quot;total_tokens&quot;: 1681,
&quot;completion_tokens&quot;: 391,
&quot;prompt_tokens_details&quot;: {
&quot;cached_tokens&quot;: 0,
&quot;image_tokens&quot;: 1240,
&quot;video_tokens&quot;: 0
},
&quot;completion_tokens_details&quot;: {
&quot;reasoning_tokens&quot;: 217,
&quot;image_tokens&quot;: 0
}
}
}
</code></pre>
<p>Example of non-streaming results with tool call, where the <code>content</code> field is empty and <code>finish_reason</code> is <code>tool_calls</code>:</p>
<pre><code class="language-json">{
&quot;choices&quot;: [
{
&quot;index&quot;: 0,
&quot;message&quot;: {
&quot;role&quot;: &quot;assistant&quot;,
&quot;content&quot;: &quot;&quot;,
&quot;multimodal_content&quot;: null,
&quot;reasoning_content&quot;: &quot;What immediately stands out is that I need to determine the spatial relationship between the old lady, the empty table, and the older couple. The original image might not provide enough detail to make this determination clearly, so I should use the image_zoom_in_tool to focus on the relevant area where these elements are located.\n&quot;,
&quot;tool_calls&quot;: [
{
&quot;id&quot;: &quot;chatcmpl-tool-dd0ef62027cf409c8f013af65f88adc3&quot;,
&quot;type&quot;: &quot;function&quot;,
&quot;function&quot;: {
&quot;name&quot;: &quot;image_zoom_in_tool&quot;,
&quot;arguments&quot;: &quot;{\&quot;bbox_2d\&quot;: [285, 235, 999, 652]}&quot;
}
}
]
}
&quot;finish_reason&quot;: &quot;tool_calls&quot;
}
],
&quot;usage&quot;: {
&quot;prompt_tokens&quot;: 280,
&quot;total_tokens&quot;: 397,
&quot;completion_tokens&quot;: 117,
&quot;prompt_tokens_details&quot;: {
&quot;cached_tokens&quot;: 0,
&quot;image_tokens&quot;: 0,
&quot;video_tokens&quot;: 0
},
&quot;completion_tokens_details&quot;: {
&quot;reasoning_tokens&quot;: 66,
&quot;image_tokens&quot;: 0
}
}
}
</code></pre>
</article>
</div>
<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
</div>
</main>
<footer class="md-footer">
<div class="md-footer-meta md-typeset">
<div class="md-footer-meta__inner md-grid">
<div class="md-copyright">
<div class="md-copyright__highlight">
Copyright &copy; 2025 Maintained by FastDeploy
</div>
Made with
<a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
Material for MkDocs
</a>
</div>
</div>
</div>
</footer>
</div>
<div class="md-dialog" data-md-component="dialog">
<div class="md-dialog__inner md-typeset"></div>
</div>
<script id="__config" type="application/json">{"annotate": null, "base": "../..", "features": [], "search": "../../assets/javascripts/workers/search.2c215733.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script>
<script src="../../assets/javascripts/bundle.79ae519e.min.js"></script>
</body>
</html>
Reference in New Issue View Git Blame Copy Permalink