huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-09 11:24:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

Skip {% generation %} and {% endgeneration %}

Browse Source 
Custom syntax within the chat template for the Phi4 Reasoning models
e.g. https://huggingface.co/microsoft/Phi-4-reasoning-plus, which is
AFAIK not handled natively yet, so skipping for now
This commit is contained in:
Alvaro Bartolome 2025-05-01 11:14:21 +02:00
parent d64d6d2f7f
commit 54cc24b3c9
No known key found for this signature in database
1 changed files with 7 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
router/src/infer/chat_template.rs
View File

@ -45,6 +45,13 @@ impl ChatTemplate {
// It uses python notation to reverse lists, which do not exist in minijinja
// so we're using the reverse filter instead.
let mutated_template = mutated_template.replace("[::-1]", "|reverse");
// TODO: replace with a better solution
// Hack to remove the {% generation %} and {% endgeneration %} statements from
// Phi4 Reasoning chat templates, as those are required when generating a mask
// for the assistant generated tokens, not natively handled yet
// Reference from Transformers at https://github.com/huggingface/transformers/blob/7a3e208892c06a5e278144eaf38c8599a42f53e7/src/transformers/processing_utils.py#L382-L385
let mutated_template = mutated_template.replace("{% generation %}", "");
let mutated_template = mutated_template.replace("{% endgeneration %}", "");
let template_str = mutated_template.into_boxed_str();
env.add_function("raise_exception", raise_exception);