From ac531c8d0aa7a991cd86d1ee301b8743d4afb55d Mon Sep 17 00:00:00 2001 From: mmnga Date: Mon, 16 Oct 2023 11:03:32 +0900 Subject: [PATCH] fix:error use loadin8bit model.cuda --- server/text_generation_server/models/causal_lm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py index fccfb0f8..8056a8ec 100644 --- a/server/text_generation_server/models/causal_lm.py +++ b/server/text_generation_server/models/causal_lm.py @@ -511,7 +511,7 @@ class CausalLM(Model): load_in_8bit=quantize == "bitsandbytes", trust_remote_code=trust_remote_code, ) - if torch.cuda.is_available() and torch.cuda.device_count() == 1: + if torch.cuda.is_available() and torch.cuda.device_count() == 1 and quantize != "bitsandbytes": model = model.cuda() if tokenizer.pad_token_id is None: