diff --git a/integration-tests/models/test_flash_gpt2.py b/integration-tests/models/test_flash_gpt2.py
index 0c7977d0..1e906521 100644
--- a/integration-tests/models/test_flash_gpt2.py
+++ b/integration-tests/models/test_flash_gpt2.py
@@ -3,7 +3,7 @@ import pytest
 
 @pytest.fixture(scope="module")
 def flash_gpt2_handle(launcher):
-    with launcher("openai-community/gpt2", num_shard=2) as handle:
+    with launcher("openai-community/gpt2", num_shard=1) as handle:
         yield handle