huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-06-19 15:52:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into fix_replaying_requests

Browse Source
This commit is contained in:
Nicolas Patry 2023-01-02 10:55:21 +01:00 committed by GitHub
commit bed5634ead
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
router/src/validation.rs
View File

@ -131,7 +131,7 @@ fn validation_worker(
}
// Get the number of tokens in the input
match tokenizer.encode(request.inputs.clone(), false) {
match tokenizer.encode(request.inputs.clone(), true) {
Ok(inputs) => {
let input_length = inputs.len();

2
server/text_generation/models/causal_lm.py
View File

@ -65,7 +65,7 @@ class CausalLMBatch:
)
all_logprobs.append(None)
pad_to_multiple_of = 8 if "gpu" in str(device) else None
pad_to_multiple_of = 8 if device.type == "cuda" else None
tokenized_inputs = tokenizer(
inputs,
return_tensors="pt",

2
server/text_generation/models/seq2seq_lm.py
View File

@ -77,7 +77,7 @@ class Seq2SeqLMBatch:
decoder_logprobs.append(None)
# Tokenize batch
pad_to_multiple_of = 8 if "gpu" in str(device) else None
pad_to_multiple_of = 8 if device.type == "cuda" else None
tokenized_inputs = tokenizer(
inputs,
return_tensors="pt",