huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-12 04:44:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: add noop in TensorParallelAdapterRowLinear too

Browse Source
This commit is contained in:
drbh 2024-06-24 22:06:04 +00:00
parent 3c9b28eaec
commit c927cffbf7
1 changed files with 3 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
server/text_generation_server/layers/lora.py
View File

@ -258,6 +258,9 @@ class TensorParallelAdapterRowLinear(LoraLinear):
) -> torch.Tensor: ) -> torch.Tensor:
result = self.base_layer(input) result = self.base_layer(input)
if self.layer_name is None:
return result
# Fused all-gather + all-reduce from S-LoRA paper: https://arxiv.org/abs/2311.03285 # Fused all-gather + all-reduce from S-LoRA paper: https://arxiv.org/abs/2311.03285
stride = result.shape[-1] // self.process_group.size() stride = result.shape[-1] // self.process_group.size()
start_idx = self.process_group.rank() * stride start_idx = self.process_group.rank() * stride