huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-27 13:02:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

(backend) use parking_lot crate for RwLock fairness

Browse Source 
# Conflicts:
#	backends/trtllm/src/backend.rs
This commit is contained in:
Morgan Funtowicz 2024-07-31 12:30:53 +00:00 committed by Morgan Funtowicz
parent 5e0fb46821
commit 25b20cba2a
1 changed files with 7 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
backends/trtllm/src/backend.rs
View File

@ -2,8 +2,8 @@ use std::future::Future;
use std::path::Path; use std::path::Path;
use std::pin::{pin, Pin}; use std::pin::{pin, Pin};
use std::str::FromStr; use std::str::FromStr;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, OnceLock}; use std::sync::{Arc, OnceLock};
use std::sync::atomic::{AtomicBool, Ordering};
use std::task::{Context, Poll}; use std::task::{Context, Poll};
use std::time::Duration; use std::time::Duration;
@ -12,17 +12,17 @@ use cxx::UniquePtr;
use log::{error, warn}; use log::{error, warn};
use tokenizers::Tokenizer; use tokenizers::Tokenizer;
use tokio::sync::mpsc::{unbounded_channel, UnboundedSender}; use tokio::sync::mpsc::{unbounded_channel, UnboundedSender};
use tokio::time::{sleep, Instant}; use tokio::time::{Instant, sleep};
use tokio_stream::wrappers::UnboundedReceiverStream;
use tokio_stream::{Stream, StreamExt}; use tokio_stream::{Stream, StreamExt};
use tracing::{instrument, span, Level}; use tokio_stream::wrappers::UnboundedReceiverStream;
use tracing::{instrument, Level, span};
// use tokio::sync::RwLock; // use tokio::sync::RwLock;
use parking_lot::RwLock; use parking_lot::RwLock;
use text_generation_router::infer::{Backend, GeneratedText, InferError, InferStreamResponse};
use text_generation_router::validation::ValidationError::UnsupportedModality;
use text_generation_router::validation::{Chunk, ValidGenerateRequest, ValidationError};
use text_generation_router::{FinishReason, Token}; use text_generation_router::{FinishReason, Token};
use text_generation_router::infer::{Backend, GeneratedText, InferError, InferStreamResponse};
use text_generation_router::validation::{Chunk, ValidationError, ValidGenerateRequest};
use text_generation_router::validation::ValidationError::UnsupportedModality;
use crate::errors::TensorRtLlmBackendError; use crate::errors::TensorRtLlmBackendError;
use crate::ffi::{create_tensorrt_llm_backend, GenerationStep, TensorRtLlmBackendImpl}; use crate::ffi::{create_tensorrt_llm_backend, GenerationStep, TensorRtLlmBackendImpl};