huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-27 04:52:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

(backend) use parking_lot crate for RwLock fairness

Browse Source 
# Conflicts:
#	backends/trtllm/src/backend.rs
This commit is contained in:
Morgan Funtowicz 2024-07-31 12:30:53 +00:00 committed by Morgan Funtowicz
parent 5e0fb46821
commit 25b20cba2a
1 changed files with 7 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
backends/trtllm/src/backend.rs
View File

@ -2,8 +2,8 @@ use std::future::Future;
use std::path::Path;
use std::pin::{pin, Pin};
use std::str::FromStr;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, OnceLock};
use std::sync::atomic::{AtomicBool, Ordering};
use std::task::{Context, Poll};
use std::time::Duration;
@ -12,17 +12,17 @@ use cxx::UniquePtr;
use log::{error, warn};
use tokenizers::Tokenizer;
use tokio::sync::mpsc::{unbounded_channel, UnboundedSender};
use tokio::time::{sleep, Instant};
use tokio_stream::wrappers::UnboundedReceiverStream;
use tokio::time::{Instant, sleep};
use tokio_stream::{Stream, StreamExt};
use tracing::{instrument, span, Level};
use tokio_stream::wrappers::UnboundedReceiverStream;
use tracing::{instrument, Level, span};
// use tokio::sync::RwLock;
use parking_lot::RwLock;
use text_generation_router::infer::{Backend, GeneratedText, InferError, InferStreamResponse};
use text_generation_router::validation::ValidationError::UnsupportedModality;
use text_generation_router::validation::{Chunk, ValidGenerateRequest, ValidationError};
use text_generation_router::{FinishReason, Token};
use text_generation_router::infer::{Backend, GeneratedText, InferError, InferStreamResponse};
use text_generation_router::validation::{Chunk, ValidationError, ValidGenerateRequest};
use text_generation_router::validation::ValidationError::UnsupportedModality;
use crate::errors::TensorRtLlmBackendError;
use crate::ffi::{create_tensorrt_llm_backend, GenerationStep, TensorRtLlmBackendImpl};