From cf2795425740645a7f8e06dc0111fcbd4439cc3e Mon Sep 17 00:00:00 2001 From: drbh Date: Fri, 2 Aug 2024 17:39:08 +0000 Subject: [PATCH] fix: update sliding window conditional --- server/text_generation_server/models/__init__.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index 40d3c07b..62687432 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -484,14 +484,13 @@ def get_model( ) sliding_window = config_dict.get("sliding_window", -1) - is_max_input_within_sliding_window = ( - max_input_tokens <= sliding_window if max_input_tokens is not None else False - ) + if max_input_tokens <= sliding_window: + sliding_window = -1 if ( (sliding_window is not None and sliding_window != -1) and not SUPPORTS_WINDOWING - and is_max_input_within_sliding_window + and max_input_tokens > sliding_window ): raise ValueError( f"The backend {SYSTEM} does not support sliding window attention that is used by the model type {model_type}. To use this model nonetheless with the {SYSTEM} backend, please launch TGI with the argument `--max-input-tokens` smaller than sliding_window={sliding_window} (got here max_input_tokens={max_input_tokens})."