diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py
index 3e181321..85f0d15c 100644
--- a/server/text_generation_server/models/__init__.py
+++ b/server/text_generation_server/models/__init__.py
@@ -147,7 +147,7 @@ def get_model(
         )
 
     elif model_type == "gpt_neox":
-        if FLASH_ATTENTION or shard:
+        if FLASH_ATTENTION:
             return FlashNeoXSharded(
                 model_id,
                 revision,