diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py
index 5b1b5715..ab3b25b7 100644
--- a/server/text_generation_server/models/__init__.py
+++ b/server/text_generation_server/models/__init__.py
@@ -87,7 +87,9 @@ def get_model(
     trust_remote_code: bool,
 ) -> Model:
     if dtype is None:
-        dtype = torch.float16
+        # Keep it as default for now and let
+        # every model resolve their own default dtype.
+        dtype = None
     elif dtype == "float16":
         dtype = torch.float16
     elif dtype == "bfloat16":