diff --git a/backends/gaudi/server/text_generation_server/tgi_service.py b/backends/gaudi/server/text_generation_server/tgi_service.py
index 18e88a7e..12317127 100644
--- a/backends/gaudi/server/text_generation_server/tgi_service.py
+++ b/backends/gaudi/server/text_generation_server/tgi_service.py
@@ -31,6 +31,7 @@ def main(args):
         trust_remote_code=args.trust_remote_code,
         uds_path=args.uds_path,
         max_input_tokens=args.max_input_tokens,
+        kv_cache_dtype="auto",
     )