From 7e20b8cb5010baf392c7bf9ef2dfb1d6ee7dd105 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Wed, 16 Aug 2023 19:58:30 +0000 Subject: [PATCH] "Fix" for rw-1b. - New "falcon" layout on this repo - No alibi - `transformers` already modifying cache layout in our stead (same modifications). - Output is garbage. Not sure why. --- server/text_generation_server/models/rw.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/server/text_generation_server/models/rw.py b/server/text_generation_server/models/rw.py index 92bb135b..d97c1c73 100644 --- a/server/text_generation_server/models/rw.py +++ b/server/text_generation_server/models/rw.py @@ -67,18 +67,6 @@ class RW(CausalLM): self, input_ids, attention_mask, position_ids, past_key_values: Optional = None ) -> Tuple[torch.Tensor, List[Tuple[torch.Tensor, torch.Tensor]]]: # Model Forward - if past_key_values is not None: - reshaped_past_key_values = [] - for layer in past_key_values: - past_keys, past_values = layer - reshaped_past_key_values.append( - ( - past_keys.view(-1, *past_keys.shape[-2:]), - past_values.view(-1, *past_values.shape[-2:]), - ) - ) - past_key_values = reshaped_past_key_values - outputs = self.model.forward( input_ids=input_ids, attention_mask=attention_mask,