From 2713b2113270716e172e6500e90f22416764d71c Mon Sep 17 00:00:00 2001
From: Nicolas Patry <patry.nicolas@protonmail.com>
Date: Mon, 27 Nov 2023 10:30:35 +0000
Subject: [PATCH] Let each model resolve their own default dtype.

---
 server/text_generation_server/models/__init__.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py
index 5b1b5715..ab3b25b7 100644
--- a/server/text_generation_server/models/__init__.py
+++ b/server/text_generation_server/models/__init__.py
@@ -87,7 +87,9 @@ def get_model(
     trust_remote_code: bool,
 ) -> Model:
     if dtype is None:
-        dtype = torch.float16
+        # Keep it as default for now and let
+        # every model resolve their own default dtype.
+        dtype = None
     elif dtype == "float16":
         dtype = torch.float16
     elif dtype == "bfloat16":