From 2804a74276c5fb6422a43bab563589d13f37ad26 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Fri, 16 Feb 2024 18:08:02 +0000 Subject: [PATCH] Fix mistral with length > window_size for long prefills (rotary doesn't create long enough cos, sin). --- server/text_generation_server/models/flash_mistral.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/text_generation_server/models/flash_mistral.py b/server/text_generation_server/models/flash_mistral.py index 70669c8d..75c3f838 100644 --- a/server/text_generation_server/models/flash_mistral.py +++ b/server/text_generation_server/models/flash_mistral.py @@ -460,8 +460,8 @@ class BaseFlashMistral(FlashCausalLM): max_s = batch.max_seqlen lm_head_indices = batch.prefill_head_indices - if self.model.max_past is not None: - max_s = min(self.model.max_past, max_s) + # if self.model.max_past is not None: + # max_s = min(self.model.max_past, max_s) bs = input_ids.shape[0] padded_bs = bs