huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-10 03:44:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

clean dtype

Browse Source
This commit is contained in:
OlivierDehaene 2023-05-12 15:53:56 +02:00
parent e7826855a3
commit b9ad3acc4e
2 changed files with 1 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
server/text_generation_server/utils/logits_process.py
View File

@ -197,7 +197,6 @@ class HeterogeneousTopKLogitsWarper(LogitsWarper):
def __init__(
self,
top_k: List[int],
dtype: torch.dtype,
device: torch.device,
filter_value: float = -math.inf,
min_tokens_to_keep: int = 1,

2
server/text_generation_server/utils/tokens.py
View File

@ -187,7 +187,7 @@ class HeterogeneousNextTokenChooser:
if any([x != 0 for x in top_k]):
do_sample = [sample or x != 0 for x, sample in zip(top_k, do_sample)]
warpers.append(HeterogeneousTopKLogitsWarper(top_k, dtype, device))
warpers.append(HeterogeneousTopKLogitsWarper(top_k, device))
if any([x < 1.0 for x in top_p]):
do_sample = [sample or x < 1.0 for x, sample in zip(top_p, do_sample)]