apps/FastDeploy
1
0
Fork 0
mirror of https://github.com/PaddlePaddle/FastDeploy.git synced 2026-04-22 16:07:51 +08:00
Code Issues Actions 15 Packages Projects Releases Wiki Activity

[Feature] Fix mixed cache-aware (#7129)

Browse Source 
* [Feature] Config eviction_duration

* [Feature] Config eviction_duration

* [Feature] Config eviction_duration

* [Feature] Config eviction_duration

* [Feature] Fix mixed cache-aware

---------

Co-authored-by: mouxin <mouxin@baidu.com>
This commit is contained in:
mouxin
2026-04-01 19:29:29 +08:00
committed by GitHub
parent 3b564116d5
commit fba8a51ad1
1 changed files with 3 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fastdeploy/golang_router/internal/gateway/completions.go
+3 -1
View File
@@ -412,7 +412,8 @@ func CommonCompletions(c *gin.Context, extractor PromptExtractor, completionEndp
} else {
logger.Info(ctx, "Parsing completed; starting worker selection.")
// Non-PD mode: use Mixed instance
dest, err := manager.SelectWorker(ctx, "")
message = extractor(rawReq)
dest, err := manager.SelectWorker(ctx, message)
if err != nil {
logger.Error(ctx, "Failed to select worker: %v", err)
c.Writer.WriteHeader(http.StatusBadGateway)
@@ -427,6 +428,7 @@ func CommonCompletions(c *gin.Context, extractor PromptExtractor, completionEndp
defer func() {
for _, url := range releaseTargets {
scheduler_handler.Release(ctx, url)
scheduler_handler.ReleasePrefillTokens(ctx, url, message)
}
}()
}