huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-22 15:32:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: add missing stop parameter for chat request (#1619)

Browse Source 
This PR adds the missing `stop` parameter to the `ChatRequest` struct
which allows calls to specify a list of stop sequences
This commit is contained in:
drbh 2024-03-01 12:08:11 -05:00 committed by Karol Damaszke
parent 5a2a0ca0c0
commit dc7c69e887
2 changed files with 7 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
router/src/lib.rs
View File

@ -611,6 +611,11 @@ pub(crate) struct ChatRequest {
#[schema(nullable = true, example = 0.1)]
pub presence_penalty: Option<f32>,
/// Up to 4 sequences where the API will stop generating further tokens.
#[serde(default)]
#[schema(nullable = true, example = "null")]
pub stop: Option<Vec<String>>,
#[serde(default = "bool::default")]
pub stream: bool,

3
router/src/server.rs
View File

@ -765,6 +765,7 @@ async fn chat_completions(
.map(|x| x + 2.0);
let logprobs = req.logprobs.unwrap_or(false);
let seed = req.seed;
let stop = req.stop.unwrap_or_default();
// apply chat template to flatten the request into a single input
let mut inputs = match infer.apply_chat_template(req.messages) {
@ -852,7 +853,7 @@ async fn chat_completions(
do_sample: true,
max_new_tokens,
return_full_text: None,
stop: Vec::new(),
stop,
truncate: None,
watermark: false,
details: true,