diff --git a/server/text_generation_server/models/custom_modeling/t5_modeling.py b/server/text_generation_server/models/custom_modeling/t5_modeling.py
index 793f3a66..a9dff6c6 100644
--- a/server/text_generation_server/models/custom_modeling/t5_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/t5_modeling.py
@@ -1033,7 +1033,7 @@ class T5ForConditionalGeneration(T5PreTrainedModel):
         )
 
         self.lm_head = TensorParallelHead.load(
-            config, prefix="lm_head", weights=weights
+            config, prefix="shared", weights=weights
         )
 
     def forward(