huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-11 20:34:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixing the text part from tokenizer endpoint.

Browse Source
This commit is contained in:
Nicolas Patry 2024-05-28 13:30:50 +00:00
parent e76b9824ae
commit 847536d9c7
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
router/src/server.rs
View File

@ -1333,7 +1333,8 @@ async fn tokenize(
.iter() .iter()
.zip(encoding.get_offsets()) .zip(encoding.get_offsets())
.map(|(&id, &(start, stop))| { .map(|(&id, &(start, stop))| {
let text: String = input.chars().skip(start).take(stop - start).collect(); let text: String =
String::from_utf8_lossy(&input.as_bytes()[start..stop]).to_string();
SimpleToken { SimpleToken {
id, id,
text, text,