huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-11 20:34:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update tensor_parallel.py

Browse Source 
Resolve the issue of abnormal conversation performance in the Baichuan large model.
This commit is contained in:
Kaixiong Happy 2024-12-03 19:00:28 +08:00 committed by GitHub
parent 2c74c55637
commit 731f890887
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 5 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
server/text_generation_server/layers/tensor_parallel.py
View File

@ -66,6 +66,11 @@ class TensorParallelHead(SuperLayer):
weight = weights.get_tensor(f"{prefix}.weight") weight = weights.get_tensor(f"{prefix}.weight")
should_gather = False should_gather = False
if config.model_type == "baichuan":
# Resolve the issue of abnormal conversation performance in the Baichuan large model.
# https://github.com/huggingface/text-generation-inference/issues/2780
weight = F.normalize(weight)
return TensorParallelHead( return TensorParallelHead(
get_linear(weight, bias=None), get_linear(weight, bias=None),
process_group=weights.process_group, process_group=weights.process_group,