From 3d81a8057751f5ccfa63126a5ac1eae0bc15b8d2 Mon Sep 17 00:00:00 2001
From: "Wang, Yi" <yi.a.wang@intel.com>
Date: Wed, 13 Mar 2024 23:19:40 +0800
Subject: [PATCH] Fix incorrect setting of max_new_tokens in warmup (#104)

Signed-off-by: Wang, Yi A <yi.a.wang@intel.com>
---
 router/client/src/client.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/router/client/src/client.rs b/router/client/src/client.rs
index 530e6df3..2bff468c 100644
--- a/router/client/src/client.rs
+++ b/router/client/src/client.rs
@@ -175,7 +175,7 @@ impl Client {
                     watermark: false,
                 }),
                 stopping_parameters: Some(StoppingCriteriaParameters {
-                    max_new_tokens: 10,
+                    max_new_tokens: cmp::min(10, max_total_tokens - max_input_length),
                     stop_sequences: vec![],
                     ignore_eos_token: true,
                 }),