fix: decrease default batch, refactors and include index in batch

launcher/src/main.rs

@@ -416,7 +416,7 @@ struct Args {
     env: bool,
 
     /// Control the maximum number of inputs that a client can send in a single request
-    #[clap(default_value = "32", long, env)]
+    #[clap(default_value = "4", long, env)]
     max_client_batch_size: usize,
 }
 

router/src/lib.rs

@@ -293,6 +293,9 @@ mod prompt_serde {
         let value = Value::deserialize(deserializer)?;
         match value {
             Value::String(s) => Ok(vec![s]),
+            Value::Array(arr) if arr.is_empty() => Err(serde::de::Error::custom(
+                "Empty array detected. Do not use an empty array for the prompt.",
+            )),
             Value::Array(arr) => arr
                 .iter()
                 .map(|v| match v {

router/src/main.rs

@@ -78,7 +78,7 @@ struct Args {
     messages_api_enabled: bool,
     #[clap(long, env, default_value_t = false)]
     disable_grammar_support: bool,
-    #[clap(default_value = "32", long, env)]
+    #[clap(default_value = "4", long, env)]
     max_client_batch_size: usize,
 }
 

router/src/server.rs

@@ -613,10 +613,10 @@ async fn completions(
         ));
     }
 
-    let mut generate_requests = Vec::new();
+    let generate_requests: Vec<GenerateRequest> = req
-    for prompt in req.prompt.iter() {
+        .prompt
-        // build the request passing some parameters
+        .iter()
-        let generate_request = GenerateRequest {
+        .map(|prompt| GenerateRequest {
             inputs: prompt.to_string(),
             parameters: GenerateParameters {
                 best_of: None,
@@ -638,9 +638,8 @@ async fn completions(
                 top_n_tokens: None,
                 grammar: None,
             },
-        };
+        })
-        generate_requests.push(generate_request);
+        .collect();
-    }
 
     let mut x_compute_type = "unknown".to_string();
     let mut x_compute_characters = 0u32;
@@ -767,23 +766,28 @@ async fn completions(
         };
 
         let sse = Sse::new(stream).keep_alive(KeepAlive::default());
-        return Ok((headers, sse).into_response());
+        Ok((headers, sse).into_response())
-    }
+    } else {
-
         let current_time = std::time::SystemTime::now()
             .duration_since(std::time::UNIX_EPOCH)
             .unwrap_or_else(|_| std::time::Duration::from_secs(0))
             .as_secs();
 
         let responses = FuturesUnordered::new();
-    for generate_request in generate_requests.into_iter() {
+        for (index, generate_request) in generate_requests.into_iter().enumerate() {
-        responses.push(generate(
+            let infer_clone = infer.clone();
-            Extension(infer.clone()),
+            let compute_type_clone = compute_type.clone();
-            Extension(compute_type.clone()),
+            let response_future = async move {
+                let result = generate(
+                    Extension(infer_clone),
+                    Extension(compute_type_clone),
                     Json(generate_request),
-        ));
+                )
+                .await;
+                result.map(|(headers, generation)| (index, headers, generation))
+            };
+            responses.push(response_future);
         }
-
         let generate_responses = responses.try_collect::<Vec<_>>().await?;
 
         let mut prompt_tokens = 0u32;
@@ -801,8 +805,7 @@ async fn completions(
 
         let choices = generate_responses
             .into_iter()
-        .enumerate()
+            .map(|(index, headers, Json(generation))| {
-        .map(|(index, (headers, Json(generation)))| {
                 let details = generation.details.unwrap_or_default();
                 if index == 0 {
                     x_compute_type = headers
@@ -868,7 +871,11 @@ async fn completions(
             object: "text_completion".to_string(),
             created: current_time,
             model: info.model_id.clone(),
-        system_fingerprint: format!("{}-{}", info.version, info.docker_label.unwrap_or("native")),
+            system_fingerprint: format!(
+                "{}-{}",
+                info.version,
+                info.docker_label.unwrap_or("native")
+            ),
             choices,
             usage: Usage {
                 prompt_tokens,
@@ -892,6 +899,7 @@ async fn completions(
 
         Ok((headers, Json(response)).into_response())
     }
+}
 
 /// Generate tokens
 #[utoipa::path(