huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-11 20:34:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Remove redundancy.

Browse Source
This commit is contained in:
Nicolas Patry 2024-09-17 17:10:30 +02:00
parent 9d702bcde3
commit b89b9fd016
No known key found for this signature in database
GPG Key ID: D2920555C90F704C
1 changed files with 2 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
router/src/server.rs
View File

@ -68,10 +68,9 @@ fn encoding_to_tokens(encoding: &tokenizers::Encoding, input: &str) -> Vec<Simpl
let offsets = encoding.get_offsets(); let offsets = encoding.get_offsets();
let input_ids = encoding.get_ids(); let input_ids = encoding.get_ids();
if offsets.len() == input_ids.len() { if offsets.len() == input_ids.len() {
encoding input_ids
.get_ids()
.iter() .iter()
.zip(encoding.get_offsets()) .zip(offsets)
.map(|(&id, &(start, stop))| { .map(|(&id, &(start, stop))| {
let text = input let text = input
.chars() .chars()