[Router] Fix semantic cache check in chat completion url (#224)

* Fix semantic cache check in chat completion url Signed-off-by: Shaoting <[email protected]> * Fix semantic cache check in chat completion url Signed-off-by: Shaoting <[email protected]> --------- Signed-off-by: Shaoting <[email protected]>
vllm-project · Mar 5, 2025 · 26fda5e · 26fda5e
1 parent 2d0617a
commit 26fda5e
Showing 1 changed file with 8 additions and 7 deletions.
diff --git a/src/vllm_router/routers/main_router.py b/src/vllm_router/routers/main_router.py
@@ -41,13 +41,14 @@
 
 @main_router.post("/v1/chat/completions")
 async def route_chat_completion(request: Request):
-    # Check if the request can be served from the semantic cache
-    logger.debug("Received chat completion request, checking semantic cache")
-    cache_response = await check_semantic_cache(request=request)
-
-    if cache_response:
-        logger.info("Serving response from semantic cache")
-        return cache_response
+    if semantic_cache_available:
+        # Check if the request can be served from the semantic cache
+        logger.debug("Received chat completion request, checking semantic cache")
+        cache_response = await check_semantic_cache(request=request)
+
+        if cache_response:
+            logger.info("Serving response from semantic cache")
+            return cache_response
 
     logger.debug("No cache hit, forwarding request to backend")
     return await route_general_request(request, "/v1/chat/completions")