huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-11 12:24:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: support do_sample param in ChatRequest

Browse Source
This commit is contained in:
drbh 2024-04-15 17:06:53 +00:00
parent 06c3d4b1ec
commit 0520bde039
2 changed files with 5 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
router/src/lib.rs
View File

@ -709,6 +709,10 @@ pub(crate) struct ChatRequest {
#[schema(nullable = true, example = "null")] #[schema(nullable = true, example = "null")]
#[serde(deserialize_with = "deserialize_tool_choice::deserialize")] #[serde(deserialize_with = "deserialize_tool_choice::deserialize")]
pub tool_choice: Option<ToolType>, pub tool_choice: Option<ToolType>,
#[serde(default)]
#[schema(default = "false", example = true)]
pub do_sample: bool,
} }
fn default_tool_prompt() -> Option<String> { fn default_tool_prompt() -> Option<String> {

2
router/src/server.rs
View File

@ -1060,7 +1060,7 @@ async fn chat_completions(
top_k: None, top_k: None,
top_p: req.top_p, top_p: req.top_p,
typical_p: None, typical_p: None,
do_sample: true, do_sample: req.do_sample,
max_new_tokens, max_new_tokens,
return_full_text: None, return_full_text: None,
stop, stop,