huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-11 12:24:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: include add_special_tokens in kserve request

Browse Source
This commit is contained in:
drbh 2024-12-19 21:53:11 +00:00
parent ab5f616920
commit c9ee5f14ed
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
router/src/kserve.rs
View File

@ -205,6 +205,7 @@ pub async fn kserve_model_infer(
let generate_request = GenerateRequest { let generate_request = GenerateRequest {
inputs: str_input.to_string(), inputs: str_input.to_string(),
parameters: payload.parameters.clone(), parameters: payload.parameters.clone(),
add_special_tokens: true,
}; };
let infer = infer.clone(); let infer = infer.clone();
let compute_type = compute_type.clone(); let compute_type = compute_type.clone();
@ -212,7 +213,7 @@ pub async fn kserve_model_infer(
async move { async move {
generate_internal(infer, compute_type, Json(generate_request), span) generate_internal(infer, compute_type, Json(generate_request), span)
.await .await
.map(|(_, Json(generation))| { .map(|(_, _, Json(generation))| {
let generation_as_bytes = generation.generated_text.as_bytes().to_vec(); let generation_as_bytes = generation.generated_text.as_bytes().to_vec();
OutputChunk { OutputChunk {
name: output.name.clone(), name: output.name.clone(),