huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-21 23:12:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

disable _custom_C.LLMM1 as it is broken for TP>=2

Browse Source
This commit is contained in:
fxmarty 2024-04-19 15:59:31 +00:00
parent 562cd4b06e
commit 81c27ba9c2
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
server/text_generation_server/utils/layers.py
View File

@ -355,7 +355,8 @@ class FastLinearROCm(nn.Module):
weight = self.weight
bias = self.bias
if IS_ROCM_SYSTEM and inp.numel() // inp.size(-1) == 1:
# TODO: fix for TP>=2, this only works for TP=1
if False and IS_ROCM_SYSTEM and inp.numel() // inp.size(-1) == 1:
batched = False
if inp.dim() == 3: