huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-12 12:54:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: adjust non tool template apply

Browse Source
This commit is contained in:
drbh 2024-08-25 19:12:59 +00:00
parent 1bf0e3b65c
commit 8b45d82897
1 changed files with 18 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
router/src/server.rs
View File

@ -2556,12 +2556,16 @@ fn prepare_chat_input(
)); ));
} }
// when response_format is set, tools are not included when applying the chat template to generate inputs
if let Some(format) = response_format { if let Some(format) = response_format {
let inputs = infer.apply_chat_template(guideline, messages, None)?; let inputs = infer.apply_chat_template(guideline, messages, None)?;
return Ok((inputs, Some(format), false)); return Ok((inputs, Some(format), false));
} }
let (updated_tools, tool_schema) = ToolGrammar::apply(tools.unwrap().clone(), tool_choice)?; // when no response_format is set and tools are included, apply the chat template with the tools
// to generate inputs
if let Some(tools) = tools {
let (updated_tools, tool_schema) = ToolGrammar::apply(tools, tool_choice)?;
let grammar = tool_schema let grammar = tool_schema
.as_ref() .as_ref()
@ -2572,8 +2576,12 @@ fn prepare_chat_input(
messages, messages,
Some((updated_tools, tool_prompt.into())), Some((updated_tools, tool_prompt.into())),
)?; )?;
return Ok((inputs, grammar, tool_schema.is_some()));
}
Ok((inputs, grammar, tool_schema.is_some())) // if no response_format or tools are set simply apply the chat template to generate inputs
let inputs = infer.apply_chat_template(guideline, messages, None)?;
Ok((inputs, None, false))
} }
#[cfg(test)] #[cfg(test)]