huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-24 00:12:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix mistralai/Mistral-7B-Instruct failed issue (#284)

Browse Source 
Signed-off-by: yuanwu <yuan.wu@intel.com>
This commit is contained in:
Yuan Wu 2025-03-06 00:01:23 +08:00 committed by GitHub
parent c35810d6f0
commit 20ea73c6d4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
server/text_generation_server/models/__init__.py
View File

@ -33,6 +33,7 @@ from text_generation_server.utils.adapter import (
from optimum.habana.transformers.modeling_utils import adapt_transformers_to_gaudi
SDP_ON_BF16 = int(os.environ.get("SDP_ON_BF16", 1))
# Disable gradients
torch.set_grad_enabled(False)
@ -49,6 +50,8 @@ def get_model(
max_input_tokens: int,
) -> Model:
adapt_transformers_to_gaudi()
if SDP_ON_BF16 == 1:
torch._C._set_math_sdp_allow_fp16_bf16_reduction(True)
if speculate is not None:
set_speculate(speculate)

1
server/text_generation_server/models/causal_lm.py
View File

@ -698,6 +698,7 @@ class CausalLM(Model):
htorch.core.hpu_set_env()
if world_size > 1:
os.environ.setdefault("DEEPSPEED_USE_HABANA_FRAMEWORKS_DETERMINISTIC_API", "1")
model = self.get_deepspeed_model(model_id, dtype, revision)
model = hq_env.prepare_model_for_quantization(model)
else: