huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-11 12:24:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix mistral with length > window_size for long prefills (rotary doesn't

Browse Source 
create long enough cos, sin).
This commit is contained in:
Nicolas Patry 2024-02-16 18:08:02 +00:00
parent 4139054b82
commit 2804a74276
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
server/text_generation_server/models/flash_mistral.py
View File

@ -460,8 +460,8 @@ class BaseFlashMistral(FlashCausalLM):
max_s = batch.max_seqlen max_s = batch.max_seqlen
lm_head_indices = batch.prefill_head_indices lm_head_indices = batch.prefill_head_indices
if self.model.max_past is not None: # if self.model.max_past is not None:
max_s = min(self.model.max_past, max_s) # max_s = min(self.model.max_past, max_s)
bs = input_ids.shape[0] bs = input_ids.shape[0]
padded_bs = bs padded_bs = bs