From d735e46ef5318347c0fef7634ea2e5485cf6a136 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Tue, 1 Oct 2024 10:52:19 +0200 Subject: [PATCH] Default dtype bfloat16. --- server/text_generation_server/models/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index 085cea5f..4dabf71d 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -1147,6 +1147,7 @@ def get_model( quantize=quantize, speculator=speculator, dtype=dtype, + default_dtype=torch.bfloat16, trust_remote_code=trust_remote_code, lora_adapter_ids=lora_adapter_ids, )