huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-12 12:54:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

(backend) use parking_lot crate for RwLock fairness

Browse Source
This commit is contained in:
Morgan Funtowicz 2024-07-31 12:30:53 +00:00
parent 34f7dcfd80
commit 4c1e234266
3 changed files with 13 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Cargo.lock generated
View File

@ -3896,6 +3896,7 @@ dependencies = [
"cxx", "cxx",
"cxx-build", "cxx-build",
"log", "log",
"parking_lot",
"pkg-config", "pkg-config",
"text-generation-router", "text-generation-router",
"thiserror", "thiserror",

5
backends/trtllm/Cargo.toml
View File

@ -8,17 +8,18 @@ homepage.workspace = true
[dependencies] [dependencies]
async-trait = "0.1" async-trait = "0.1"
async-stream = "0.3" async-stream = "0.3"
clap = { version = "4.5", features = ["derive"] }
cxx = "1.0" cxx = "1.0"
log = { version = "0.4", features = [] }
text-generation-router = { path = "../../router" } text-generation-router = { path = "../../router" }
tokenizers = { version = "0.19", features = ["hf-hub"] } tokenizers = { version = "0.19", features = ["hf-hub"] }
tokio = { version = "1.38", features = ["rt", "rt-multi-thread", "parking_lot", "signal", "sync"] } tokio = { version = "1.38", features = ["rt", "rt-multi-thread", "parking_lot", "signal", "sync"] }
tokio-stream = "0.1.15" tokio-stream = "0.1.15"
clap = { version = "4.5", features = ["derive"] }
thiserror = "1.0.62" thiserror = "1.0.62"
tracing = "0.1" tracing = "0.1"
tracing-opentelemetry = "0.24" tracing-opentelemetry = "0.24"
tracing-subscriber = { version = "0.3", features = ["json", "env-filter"] } tracing-subscriber = { version = "0.3", features = ["json", "env-filter"] }
log = { version = "0.4", features = [] } parking_lot = "0.12"
[build-dependencies] [build-dependencies]
cmake = "0.1" cmake = "0.1"

17
backends/trtllm/src/backend.rs
View File

@ -2,8 +2,8 @@ use std::future::Future;
use std::path::Path; use std::path::Path;
use std::pin::{pin, Pin}; use std::pin::{pin, Pin};
use std::str::FromStr; use std::str::FromStr;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, OnceLock}; use std::sync::{Arc, OnceLock};
use std::sync::atomic::{AtomicBool, Ordering};
use std::task::{Context, Poll}; use std::task::{Context, Poll};
use std::time::Duration; use std::time::Duration;
@ -12,16 +12,17 @@ use cxx::UniquePtr;
use log::{error, warn}; use log::{error, warn};
use tokenizers::Tokenizer; use tokenizers::Tokenizer;
use tokio::sync::mpsc::{unbounded_channel, UnboundedSender}; use tokio::sync::mpsc::{unbounded_channel, UnboundedSender};
use tokio::sync::RwLock; use tokio::time::{Instant, sleep};
use tokio::time::{sleep, Instant};
use tokio_stream::wrappers::UnboundedReceiverStream;
use tokio_stream::{Stream, StreamExt}; use tokio_stream::{Stream, StreamExt};
use tracing::{instrument, span, Level}; use tokio_stream::wrappers::UnboundedReceiverStream;
use tracing::{instrument, Level, span};
use text_generation_router::infer::{Backend, GeneratedText, InferError, InferStreamResponse}; // use tokio::sync::RwLock;
use text_generation_router::validation::ValidationError::UnsupportedModality; use parking_lot::RwLock;
use text_generation_router::validation::{Chunk, ValidGenerateRequest, ValidationError};
use text_generation_router::{FinishReason, Token}; use text_generation_router::{FinishReason, Token};
use text_generation_router::infer::{Backend, GeneratedText, InferError, InferStreamResponse};
use text_generation_router::validation::{Chunk, ValidationError, ValidGenerateRequest};
use text_generation_router::validation::ValidationError::UnsupportedModality;
use crate::errors::TensorRtLlmBackendError; use crate::errors::TensorRtLlmBackendError;
use crate::ffi::{create_tensorrt_llm_backend, GenerationStep, TensorRtLlmBackendImpl}; use crate::ffi::{create_tensorrt_llm_backend, GenerationStep, TensorRtLlmBackendImpl};