From d77f5f2efff26b2374f66a03dcb068453ea857f9 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Thu, 29 Aug 2024 11:59:31 +0200 Subject: [PATCH] Update server/text_generation_server/layers/attention/common.py Co-authored-by: OlivierDehaene --- server/text_generation_server/layers/attention/common.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/text_generation_server/layers/attention/common.py b/server/text_generation_server/layers/attention/common.py index 0faec0ea..855f4dfc 100644 --- a/server/text_generation_server/layers/attention/common.py +++ b/server/text_generation_server/layers/attention/common.py @@ -38,8 +38,6 @@ if ATTENTION in {"flashinfer", "flashdecoding"}: assert max_q is not None assert max_k is not None cu_seqlen_k = torch.zeros(shape[-1] + 1, device=device, dtype=torch.int32) - # if max_q is not None and max_q < 1000 and max_q > 1: - # import ipdb;ipdb.set_trace() # cuda graphs don't like this and this is necessary to clamp within mistral # Although FA2 might not want the clamping