huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-12 04:44:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

revert default dtype

Browse Source
This commit is contained in:
OlivierDehaene 2024-07-22 16:13:53 +02:00
parent 0d68619efa
commit 6d8e3659a9
No known key found for this signature in database
GPG Key ID: BB104D67809DA93C
1 changed files with 3 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
server/text_generation_server/models/__init__.py
View File

@ -334,13 +334,6 @@ def get_model(
if FBGEMM_DYN_AVAILABLE:
# fbgemm kernels are fp8xfp8->bf16
dtype = torch.bfloat16
else:
config_dtype = config_dict.get("torch_dtype", None)
# Only use the config dtype if its one of TGI's supported dtype
if config_dtype == "float16":
dtype = torch.float16
elif config_dtype == "bfloat16":
dtype = torch.bfloat16
else:
# Keep it as default for now and let
# every model resolve their own default dtype.