From 14e384909586faae4eebcad1da1a48bb2c14ce22 Mon Sep 17 00:00:00 2001 From: Shaoting Date: Thu, 27 Feb 2025 13:52:11 -0600 Subject: [PATCH] [Bugfix] Update QPS when no requests (#174) Signed-off-by: Shaoting Feng --- src/vllm_router/request_stats.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/vllm_router/request_stats.py b/src/vllm_router/request_stats.py index 37b7ffe1..16a43fb0 100644 --- a/src/vllm_router/request_stats.py +++ b/src/vllm_router/request_stats.py @@ -71,6 +71,17 @@ def update(self, timestamp: float, value: float): self.timestamps.popleft() self.values.popleft() + def update_no_value(self, timestamp: float): + """ + Update the throughput monitor with a new timestamp with no value + """ + while ( + len(self.timestamps) > 0 + and self.timestamps[0] < timestamp - self.sliding_window_size + ): + self.timestamps.popleft() + self.values.popleft() + def get_average(self) -> float: return sum(self.values) / len(self.values) if self.values else -1 @@ -221,11 +232,15 @@ def get_request_stats(self, current_time: float) -> Dict[str, RequestStats]: if engine_url not in self.qps_monitors: qps = -1 else: + # Update the monitors + self.qps_monitors[engine_url].update_no_value(current_time) qps = self.qps_monitors[engine_url].get_sum() / self.sliding_window_size if engine_url not in self.ttft_monitors: ttft = -1 else: + # Update the monitors + self.ttft_monitors[engine_url].update_no_value(current_time) ttft = self.ttft_monitors[engine_url].get_average() in_prefill = self.in_prefill_requests.get(engine_url, 0)