huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-11 20:34:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: improve infer and simplify

Browse Source
This commit is contained in:
drbh 2024-05-23 16:21:42 +00:00
parent 0f1c4b12ca
commit 01bd1b2c26
1 changed files with 13 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
router/src/server.rs
View File

@ -1489,12 +1489,20 @@ async fn post_v2_models_model_name_versions_model_version_infer(
})
.collect::<Result<Vec<_>, _>>()?;
let output_chunks = payload
.inputs
if str_inputs.len() != payload.outputs.len() {
return Err((
StatusCode::UNPROCESSABLE_ENTITY,
Json(ErrorResponse {
error: "Inputs and outputs length mismatch".to_string(),
error_type: "length mismatch".to_string(),
}),
));
}
let output_chunks = str_inputs
.iter()
.zip(&payload.outputs)
.zip(&str_inputs)
.map(|((input, output), str_input)| {
.map(|(str_input, output)| {
let generate_request = GenerateRequest {
inputs: str_input.to_string(),
parameters: payload.parameters.clone(),
@ -1509,7 +1517,7 @@ async fn post_v2_models_model_name_versions_model_version_infer(
let generation_as_bytes = generation.generated_text.as_bytes().to_vec();
OutputChunk {
name: output.name.clone(),
shape: input.shape.clone(),
shape: vec![1, generation_as_bytes.len()],
datatype: "BYTES".to_string(),
data: generation_as_bytes,
}