huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-26 04:22:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

fixed SSE naming (#61)

Browse Source 
https://en.wikipedia.org/wiki/Server-sent_events
This commit is contained in:
Yannic Kilcher 2023-02-08 22:30:11 +01:00 committed by GitHub
parent 1ad3250b89
commit e520d5b349
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 5 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -38,7 +38,7 @@ to power LLMs api-inference widgets.
## Features ## Features
- Token streaming using Server Side Events (SSE) - Token streaming using Server-Sent Events (SSE)
- [Dynamic batching of incoming requests](https://github.com/huggingface/text-generation-inference/blob/main/router/src/batcher.rs#L88) for increased total throughput - [Dynamic batching of incoming requests](https://github.com/huggingface/text-generation-inference/blob/main/router/src/batcher.rs#L88) for increased total throughput
- Quantization with [bitsandbytes](https://github.com/TimDettmers/bitsandbytes) - Quantization with [bitsandbytes](https://github.com/TimDettmers/bitsandbytes)
- [Safetensors](https://github.com/huggingface/safetensors) weight loading - [Safetensors](https://github.com/huggingface/safetensors) weight loading

4
docs/openapi.json
View File

@ -119,8 +119,8 @@
"tags": [ "tags": [
"Text Generation Inference" "Text Generation Inference"
], ],
"summary": "Generate a stream of token using Server Side Events", "summary": "Generate a stream of token using Server-Sent Events",
"description": "Generate a stream of token using Server Side Events", "description": "Generate a stream of token using Server-Sent Events",
"operationId": "generate_stream", "operationId": "generate_stream",
"requestBody": { "requestBody": {
"content": { "content": {

4
router/src/server.rs
View File

@ -151,7 +151,7 @@ async fn generate(
Ok((headers, Json(response))) Ok((headers, Json(response)))
} }
/// Generate a stream of token using Server Side Events /// Generate a stream of token using Server-Sent Events
#[utoipa::path( #[utoipa::path(
post, post,
tag = "Text Generation Inference", tag = "Text Generation Inference",
@ -199,7 +199,7 @@ async fn generate_stream(
match infer.generate_stream(req.0).await { match infer.generate_stream(req.0).await {
Ok(mut response_stream) => { Ok(mut response_stream) => {
// Server Side Event stream // Server-Sent Event stream
while let Some(response) = response_stream.next().await { while let Some(response) = response_stream.next().await {
match response { match response {
Ok(response) => { Ok(response) => {