From eef3c7bdf2e5c6018988ae362ad59b11405a6c1e Mon Sep 17 00:00:00 2001 From: drbh Date: Thu, 23 Jan 2025 15:07:19 +0000 Subject: [PATCH] fix: prefer default dtype --- server/text_generation_server/models/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index 85c98bfd..f8150b5e 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -1362,7 +1362,8 @@ def get_model( revision=revision, quantize=quantize, speculator=speculator, - dtype=torch.bfloat16, + dtype=dtype, + default_dtype=torch.bfloat16, kv_cache_dtype=kv_cache_dtype, trust_remote_code=trust_remote_code, lora_adapter_ids=lora_adapter_ids,