huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-24 16:32:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Reducing number of reps while autotuning.

Browse Source
This commit is contained in:
Ubuntu 2023-06-06 11:56:10 +00:00
parent 7de104b7f6
commit fb0840944c
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
server/text_generation_server/quant/custom_autotune.py
View File

@ -69,7 +69,7 @@ class Autotuner(triton.KernelInterface):
try: try:
# In testings using only 40 reps seems to be close enough and it appears to be what PyTorch uses # In testings using only 40 reps seems to be close enough and it appears to be what PyTorch uses
# PyTorch also sets fast_flush to True, but I didn't see any speedup so I'll leave the default # PyTorch also sets fast_flush to True, but I didn't see any speedup so I'll leave the default
return triton.testing.do_bench(kernel_call, percentiles=(0.5, 0.2, 0.8), rep=40) return triton.testing.do_bench(kernel_call, percentiles=(0.5, 0.2, 0.8), rep=10)
except triton.compiler.OutOfResources: except triton.compiler.OutOfResources:
return (float('inf'), float('inf'), float('inf')) return (float('inf'), float('inf'), float('inf'))