diff --git a/router/src/infer.rs b/router/src/infer.rs index 224d4c5d..6de07982 100644 --- a/router/src/infer.rs +++ b/router/src/infer.rs @@ -127,7 +127,7 @@ impl Infer { // Append the request to the queue self.queue.append(Entry { - request: valid_request.clone(), + request: valid_request, response_tx, span: Span::current(), temp_span: None, diff --git a/router/src/server.rs b/router/src/server.rs index edb4f718..fe1827c4 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -401,6 +401,7 @@ async fn generate_stream_internal( match infer.generate_stream(req).instrument(info_span!(parent: &span, "async_stream")).await { // Keep permit as long as generate_stream lives Ok((_permit, _input_length, mut response_stream)) => { + let mut index = 0; // Server-Sent Event stream while let Some(response) = response_stream.next().await { index += 1;