huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-06-19 15:52:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

unconditionally call InitializeBackend on the FFI layer

Browse Source
This commit is contained in:
Morgan Funtowicz 2024-07-08 22:09:09 +00:00
parent da926feaa1
commit 68a0247a2c
1 changed files with 3 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
backends/trtllm/src/ffi.cpp
View File

@ -13,6 +13,9 @@ namespace huggingface::tgi::backends {
* @return
*/
std::unique_ptr<TensorRtLlmBackend> create_trtllm_backend(rust::Str engineFolder, rust::Str executorWorker) {
// Unconditionally call this to initialize and discover TRTLLM plugins
InitializeBackend();
const auto enginePath = std::string_view(engineFolder.begin(), engineFolder.end());
const auto executorPath = std::string_view(executorWorker.begin(), executorWorker.end());
return std::make_unique<TensorRtLlmBackend>(std::move(enginePath), std::move(executorPath));