From 8209cb90b214dc93d333dbd0cdd2bb0c187f95a7 Mon Sep 17 00:00:00 2001 From: "Wang, Yi A" Date: Tue, 20 May 2025 04:41:11 -0700 Subject: [PATCH] fix the crash in default ATTENTION path Signed-off-by: Wang, Yi A --- backends/gaudi/server/text_generation_server/tgi_service.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backends/gaudi/server/text_generation_server/tgi_service.py b/backends/gaudi/server/text_generation_server/tgi_service.py index 18e88a7e..12317127 100644 --- a/backends/gaudi/server/text_generation_server/tgi_service.py +++ b/backends/gaudi/server/text_generation_server/tgi_service.py @@ -31,6 +31,7 @@ def main(args): trust_remote_code=args.trust_remote_code, uds_path=args.uds_path, max_input_tokens=args.max_input_tokens, + kv_cache_dtype="auto", )