diff --git a/server/text_generation_server/cli.py b/server/text_generation_server/cli.py
index c8f40c46..b741a84c 100644
--- a/server/text_generation_server/cli.py
+++ b/server/text_generation_server/cli.py
@@ -150,15 +150,16 @@ def download_weights(
             if not extension == ".safetensors" or not auto_convert:
                 raise e
 
-    # Try to load as a local PEFT model
-    try:
-        utils.download_and_unload_peft(
-            model_id, revision, trust_remote_code=trust_remote_code
-        )
-        utils.weight_files(model_id, revision, extension)
-        return
-    except (utils.LocalEntryNotFoundError, utils.EntryNotFoundError):
-        pass
+    else:
+        # Try to load as a local PEFT model
+        try:
+            utils.download_and_unload_peft(
+                model_id, revision, trust_remote_code=trust_remote_code
+            )
+            utils.weight_files(model_id, revision, extension)
+            return
+        except (utils.LocalEntryNotFoundError, utils.EntryNotFoundError):
+            pass
 
     # Try to see if there are local pytorch weights
     try: