From 43b1b07fb96322f02b9c6af76c6aad721ab1729a Mon Sep 17 00:00:00 2001 From: "Wang, Yi" Date: Tue, 20 May 2025 20:02:32 +0800 Subject: [PATCH] Fix the crash in default ATTENTION path for Gaudi backend (#3235) Signed-off-by: Wang, Yi A --- backends/gaudi/server/text_generation_server/tgi_service.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backends/gaudi/server/text_generation_server/tgi_service.py b/backends/gaudi/server/text_generation_server/tgi_service.py index 18e88a7e..12317127 100644 --- a/backends/gaudi/server/text_generation_server/tgi_service.py +++ b/backends/gaudi/server/text_generation_server/tgi_service.py @@ -31,6 +31,7 @@ def main(args): trust_remote_code=args.trust_remote_code, uds_path=args.uds_path, max_input_tokens=args.max_input_tokens, + kv_cache_dtype="auto", )