huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-10 20:04:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Revert "Idefics force bfloat16"

Browse Source 
This reverts commit b8952b2b32.
This commit is contained in:
Nicolas Patry 2023-11-23 13:57:02 +00:00
parent b8952b2b32
commit 861acdeab1
1 changed files with 1 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
server/text_generation_server/models/idefics.py
View File

@ -39,8 +39,7 @@ class IDEFICSSharded(IdeficsCausalLM):
device = torch.device(f"cuda:{rank}")
# 9b seems to work correctly enough in float16, but 80b seems
# to be really saturating for f16.
if dtype is None or dtype == torch.float16:
dtype = torch.bfloat16
dtype = torch.bfloat16 if dtype is None else dtype
else:
device = torch.device("cpu")
dtype = torch.float32 if dtype is None else dtype