huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-11 04:14:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

remove v2 for now

Browse Source
This commit is contained in:
OlivierDehaene 2023-12-21 15:24:41 +01:00
parent 672f290901
commit 96a520ec78
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
server/text_generation_server/utils/layers.py
View File

@ -33,7 +33,7 @@ except Exception:
HAS_EXLLAMA = False
CAN_EXLLAMA = major >= 8
V2 = os.getenv("EXLLAMA_VERSION", "2") == "2"
V2 = os.getenv("EXLLAMA_VERSION", "1") == "2"
if V2 and int(os.getenv("WORLD_SIZE", "1")) > 1:
logger.warning(
"Disabling exllama v2 and using v1 instead because there are issues when sharding"