huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-06-19 15:52:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

make sure executor_worker is provided

Browse Source
This commit is contained in:
Morgan Funtowicz 2024-07-19 11:56:43 +00:00
parent 97723d1458
commit 6300bab8b4
2 changed files with 9 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
backends/trtllm/src/backend.rs
View File

@ -103,13 +103,13 @@ impl TensorRtLlmBackend {
pub fn new<P: AsRef<Path> + Send + 'static, PP: AsRef<Path> + Send + 'static>( pub fn new<P: AsRef<Path> + Send + 'static, PP: AsRef<Path> + Send + 'static>(
tokenizer: Tokenizer, tokenizer: Tokenizer,
engine_folder: P, engine_folder: P,
_executor_worker_path: Option<PP>, executor_worker_path: PP,
) -> Result<Self, TensorRtLlmBackendError> { ) -> Result<Self, TensorRtLlmBackendError> {
Ok(TensorRtLlmBackend { Ok(TensorRtLlmBackend {
tokenizer: Arc::new(tokenizer), tokenizer: Arc::new(tokenizer),
backend: Arc::new(RwLock::new(create_tensorrt_llm_backend( backend: Arc::new(RwLock::new(create_tensorrt_llm_backend(
engine_folder.as_ref().to_str().unwrap(), engine_folder.as_ref().to_str().unwrap(),
"", executor_worker_path.as_ref().to_str().unwrap(),
))), ))),
}) })
} }

10
backends/trtllm/src/main.rs
View File

@ -56,12 +56,8 @@ struct Args {
max_client_batch_size: usize, max_client_batch_size: usize,
#[clap(long, env)] #[clap(long, env)]
auth_token: Option<String>, auth_token: Option<String>,
#[clap( #[clap(long, env, help = "Path to the TensorRT-LLM Orchestrator worker")]
long, executor_worker: PathBuf,
env,
help = "Path to the TensorRT-LLM Orchestrator Worker binary"
)]
executor_worker: Option<PathBuf>,
} }
#[tokio::main] #[tokio::main]
@ -123,14 +119,12 @@ async fn main() -> Result<(), TensorRtLlmBackendError> {
} }
} }
if let Some(ref executor_worker) = executor_worker {
if !executor_worker.exists() { if !executor_worker.exists() {
return Err(TensorRtLlmBackendError::ArgumentValidation(format!( return Err(TensorRtLlmBackendError::ArgumentValidation(format!(
"`executor_work` specified path doesn't exists: {}", "`executor_work` specified path doesn't exists: {}",
executor_worker.display() executor_worker.display()
))); )));
} }
}
// Run server // Run server
let tokenizer = Tokenizer::from_pretrained( let tokenizer = Tokenizer::from_pretrained(