From c927cffbf77534e2def5bc09ce170eb7995cd296 Mon Sep 17 00:00:00 2001 From: drbh Date: Mon, 24 Jun 2024 22:06:04 +0000 Subject: [PATCH] fix: add noop in TensorParallelAdapterRowLinear too --- server/text_generation_server/layers/lora.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/text_generation_server/layers/lora.py b/server/text_generation_server/layers/lora.py index b6f005ab..36fdf42d 100644 --- a/server/text_generation_server/layers/lora.py +++ b/server/text_generation_server/layers/lora.py @@ -258,6 +258,9 @@ class TensorParallelAdapterRowLinear(LoraLinear): ) -> torch.Tensor: result = self.base_layer(input) + if self.layer_name is None: + return result + # Fused all-gather + all-reduce from S-LoRA paper: https://arxiv.org/abs/2311.03285 stride = result.shape[-1] // self.process_group.size() start_idx = self.process_group.rank() * stride