huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-11 12:24:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

Remove unecessary cuda graph.

Browse Source
This commit is contained in:
Nicolas Patry 2024-03-21 20:09:28 +01:00
parent de6cb15fa5
commit 2e754ffd2e
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
server/text_generation_server/models/flash_causal_lm.py
View File

@ -802,7 +802,7 @@ class FlashCausalLM(Model):
try:
logger.info("Experimental support for Cuda Graphs is enabled")
# Warmup cuda graphs
for bs in [1, 2, 4] + [8 * i for i in range(8)]:
for bs in [1, 2, 4] + [8 * i for i in range(1, 9)]:
if self.speculate is None or self.speculate + 1 <= bs:
self.cuda_graph_warmup(bs, max_s, max_bt)
except Exception: