diff --git a/Dockerfile_gaudi b/Dockerfile_gaudi
index 06073fe4..5bb5b606 100644
--- a/Dockerfile_gaudi
+++ b/Dockerfile_gaudi
@@ -60,6 +60,7 @@ FROM vault.habana.ai/gaudi-docker/${HABANA_VERSION}/ubuntu22.04/habanalabs/pytor
 ENV ATTENTION=default
 ENV PREFIX_CACHING=0
 ENV PREFILL_CHUNKING=0
+ENV PT_HPU_LAZY_MODE=1
 
 # Text Generation Inference base env
 ENV HF_HOME=/data \