From 0520bde039a9b38abaf91c47dd9e3f58e6e30907 Mon Sep 17 00:00:00 2001
From: drbh <david.richard.holtz@gmail.com>
Date: Mon, 15 Apr 2024 17:06:53 +0000
Subject: [PATCH] feat: support do_sample param in ChatRequest

---
 router/src/lib.rs    | 4 ++++
 router/src/server.rs | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/router/src/lib.rs b/router/src/lib.rs
index 2395e3e2..17ab00e7 100644
--- a/router/src/lib.rs
+++ b/router/src/lib.rs
@@ -709,6 +709,10 @@ pub(crate) struct ChatRequest {
     #[schema(nullable = true, example = "null")]
     #[serde(deserialize_with = "deserialize_tool_choice::deserialize")]
     pub tool_choice: Option<ToolType>,
+
+    #[serde(default)]
+    #[schema(default = "false", example = true)]
+    pub do_sample: bool,
 }
 
 fn default_tool_prompt() -> Option<String> {
diff --git a/router/src/server.rs b/router/src/server.rs
index 302a4753..e728455d 100644
--- a/router/src/server.rs
+++ b/router/src/server.rs
@@ -1060,7 +1060,7 @@ async fn chat_completions(
             top_k: None,
             top_p: req.top_p,
             typical_p: None,
-            do_sample: true,
+            do_sample: req.do_sample,
             max_new_tokens,
             return_full_text: None,
             stop,