huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-11 12:24:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: defaults add_generation_prompt true

Browse Source
This commit is contained in:
drbh 2024-02-06 09:08:35 -05:00
parent 76093c79ac
commit ff0428a351
2 changed files with 3 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
router/src/infer.rs
View File

@ -38,7 +38,6 @@ pub struct Infer {
Option<Template<'static, 'static>>, Option<Template<'static, 'static>>,
Option<String>, Option<String>,
Option<String>, Option<String>,
bool,
), ),
} }
@ -107,13 +106,12 @@ impl Infer {
.bos_token .bos_token
.map_or_else(String::new, |t| t) .map_or_else(String::new, |t| t)
.into(); .into();
let add_generation_prompt = tokenizer_config.add_generation_prompt.unwrap_or(false);
Self { Self {
validation, validation,
queue, queue,
shared, shared,
limit_concurrent_requests: semaphore, limit_concurrent_requests: semaphore,
template: (template, eos_token, bos_token, add_generation_prompt), template: (template, eos_token, bos_token),
} }
} }
@ -192,7 +190,7 @@ impl Infer {
/// Apply the chat template to the chat request /// Apply the chat template to the chat request
#[instrument(skip_all)] #[instrument(skip_all)]
pub(crate) fn apply_chat_template(&self, messages: Vec<Message>) -> Result<String, InferError> { pub(crate) fn apply_chat_template(&self, messages: Vec<Message>) -> Result<String, InferError> {
let (template, bos_token, eos_token, add_generation_prompt) = &self.template; let (template, bos_token, eos_token) = &self.template;
template template
.as_ref() .as_ref()
.ok_or_else(|| InferError::TemplateError(ErrorKind::TemplateNotFound.into()))? .ok_or_else(|| InferError::TemplateError(ErrorKind::TemplateNotFound.into()))?
@ -200,7 +198,7 @@ impl Infer {
messages, messages,
eos_token: eos_token.as_deref(), eos_token: eos_token.as_deref(),
bos_token: bos_token.as_deref(), bos_token: bos_token.as_deref(),
add_generation_prompt: *add_generation_prompt, add_generation_prompt: true,
}) })
.map_err(|e| { .map_err(|e| {
metrics::increment_counter!("tgi_request_failure", "err" => "template"); metrics::increment_counter!("tgi_request_failure", "err" => "template");

1
router/src/lib.rs
View File

@ -34,7 +34,6 @@ pub struct HubTokenizerConfig {
pub chat_template: Option<String>, pub chat_template: Option<String>,
pub bos_token: Option<String>, pub bos_token: Option<String>,
pub eos_token: Option<String>, pub eos_token: Option<String>,
pub add_generation_prompt: Option<bool>,
} }
impl HubTokenizerConfig { impl HubTokenizerConfig {