huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-07-29 19:30:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: prefer default dtype

Browse Source
This commit is contained in:
drbh 2025-01-23 15:07:19 +00:00
parent 7ab99bc6b3
commit eef3c7bdf2
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
server/text_generation_server/models/__init__.py
View File

@ -1362,7 +1362,8 @@ def get_model(
revision=revision, revision=revision,
quantize=quantize, quantize=quantize,
speculator=speculator, speculator=speculator,
dtype=torch.bfloat16, dtype=dtype,
default_dtype=torch.bfloat16,
kv_cache_dtype=kv_cache_dtype, kv_cache_dtype=kv_cache_dtype,
trust_remote_code=trust_remote_code, trust_remote_code=trust_remote_code,
lora_adapter_ids=lora_adapter_ids, lora_adapter_ids=lora_adapter_ids,