huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-11 04:14:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

back to v2 by def

Browse Source
This commit is contained in:
OlivierDehaene 2023-12-21 15:46:07 +01:00
parent 96a520ec78
commit 238cc311f1
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
server/text_generation_server/utils/layers.py
View File

@ -33,7 +33,7 @@ except Exception:
HAS_EXLLAMA = False HAS_EXLLAMA = False
CAN_EXLLAMA = major >= 8 CAN_EXLLAMA = major >= 8
V2 = os.getenv("EXLLAMA_VERSION", "1") == "2" V2 = os.getenv("EXLLAMA_VERSION", "2") == "2"
if V2 and int(os.getenv("WORLD_SIZE", "1")) > 1: if V2 and int(os.getenv("WORLD_SIZE", "1")) > 1:
logger.warning( logger.warning(
"Disabling exllama v2 and using v1 instead because there are issues when sharding" "Disabling exllama v2 and using v1 instead because there are issues when sharding"