From 5c72f269b64dca78f388e000741c0ae9f9f1e62f Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Wed, 16 Oct 2024 10:46:03 +0200 Subject: [PATCH] Fix prefix_caching variable, remove defaults in server (confusing a lot of the times). --- server/text_generation_server/models/globals.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/text_generation_server/models/globals.py b/server/text_generation_server/models/globals.py index 8be92fbf..4ac6a6b4 100644 --- a/server/text_generation_server/models/globals.py +++ b/server/text_generation_server/models/globals.py @@ -5,9 +5,9 @@ from typing import Dict, Optional from text_generation_server.utils.log import log_master -ATTENTION = os.getenv("ATTENTION", "flashinfer") -default_prefix_caching = "1" if ATTENTION in {"flashinfer", "flashdecoding"} else "0" -PREFIX_CACHING = os.getenv("PREFIX_CACHING", default_prefix_caching).lower() in { +ATTENTION = os.environ["ATTENTION"] +# default_prefix_caching = "1" if ATTENTION in {"flashinfer", "flashdecoding"} else "0" +PREFIX_CACHING = os.environ["PREFIX_CACHING"].lower() in { "1", "true", }