From 40dfce644a4e5fe08652e9e7f02cbb9d67f8e0b0 Mon Sep 17 00:00:00 2001
From: Alvaro Bartolome <36760800+alvarobartt@users.noreply.github.com>
Date: Thu, 1 May 2025 12:13:17 +0200
Subject: [PATCH] Skip `{% generation %}` and `{% endgeneration %}` template
 handling (#3204)

* Add `.DS_Store` file to `.gitignore`

* Skip `{% generation %}` and `{% endgeneration %}`

Custom syntax within the chat template for the Phi4 Reasoning models
e.g. https://huggingface.co/microsoft/Phi-4-reasoning-plus, which is
AFAIK not handled natively yet, so skipping for now

* Update explanation on `{% generation %}` and `{% endgeneration %}` removal

* Revert "Add `.DS_Store` file to `.gitignore`"

This reverts commit d64d6d2f7f5283b74f1c9aa906d6e68e37edfb48.
---
 router/src/infer/chat_template.rs | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/router/src/infer/chat_template.rs b/router/src/infer/chat_template.rs
index b26b3824..4d28956f 100644
--- a/router/src/infer/chat_template.rs
+++ b/router/src/infer/chat_template.rs
@@ -45,6 +45,12 @@ impl ChatTemplate {
         //  It uses python notation to reverse lists, which do not exist in minijinja
         //  so we're using the reverse filter instead.
         let mutated_template = mutated_template.replace("[::-1]", "|reverse");
+        // TODO: replace with a better solution
+        // Hack to remove the {% generation %} and {% endgeneration %} statements from
+        // the Jinja2 chat templates if there, since those are only using for assistant
+        // masking during training, and should be ignored during inference
+        let mutated_template = mutated_template.replace("{% generation %}", "");
+        let mutated_template = mutated_template.replace("{% endgeneration %}", "");
 
         let template_str = mutated_template.into_boxed_str();
         env.add_function("raise_exception", raise_exception);