From 06b638f3108d4c28c1a31539f76f9f6f3dcb33fb Mon Sep 17 00:00:00 2001 From: drbh Date: Thu, 8 Aug 2024 11:14:06 -0400 Subject: [PATCH] Pr 2374 ci branch (#2378) * Update __init__.py Fix issue with NoneType comparison for max_input_tokens and sliding_window - Add default values for max_input_tokens and sliding_window to handle None cases. - Ensure the comparison between max_input_tokens and sliding_window is handled correctly to prevent TypeError. - This change addresses the error: TypeError: '<=' not supported between instances of 'int' and 'NoneType'. * Update __init__.py Handle NoneType in sliding_window comparison to fix TypeError in __init__.py by ensuring the comparison logic accounts for NoneType values, preventing errors and improving code robustness. * fix: syntax/style tweak --------- Co-authored-by: Praz --- server/text_generation_server/models/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index 1f9c75266..da14d083f 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -490,7 +490,12 @@ def get_model( raise RuntimeError( "Sharding is currently not supported with `exl2` quantization" ) - sliding_window = config_dict.get("sliding_window", -1) + + sliding_window = ( + config_dict.get("sliding_window") + if config_dict.get("sliding_window") is not None + else -1 + ) if max_input_tokens is not None and max_input_tokens <= sliding_window: sliding_window = -1