huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-11 04:14:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Idefics force bfloat16

Browse Source
This commit is contained in:
Vince Jankovics 2023-10-31 12:51:13 +00:00
parent 96a982ad8f
commit b8952b2b32
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
server/text_generation_server/models/idefics.py
View File

@ -39,7 +39,8 @@ class IDEFICSSharded(IdeficsCausalLM):
device = torch.device(f"cuda:{rank}")
# 9b seems to work correctly enough in float16, but 80b seems
# to be really saturating for f16.
dtype = torch.bfloat16 if dtype is None else dtype
if dtype is None or dtype == torch.float16:
dtype = torch.bfloat16
else:
device = torch.device("cpu")
dtype = torch.float32 if dtype is None else dtype