From 848b8ad554ff7d582bfbfb69ff0bcacf18a72ce8 Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Mon, 21 Oct 2024 23:40:20 +0200 Subject: [PATCH] chore(trtllm): minor refactoring --- backends/trtllm/lib/backend.cpp | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/backends/trtllm/lib/backend.cpp b/backends/trtllm/lib/backend.cpp index 6a4a7be7c..8a4ac4c04 100644 --- a/backends/trtllm/lib/backend.cpp +++ b/backends/trtllm/lib/backend.cpp @@ -100,11 +100,10 @@ huggingface::tgi::backends::TensorRtLlmBackend::TensorRtLlmBackend( SPDLOG_INFO(FMT_STRING("Found {:d} EOS tokens"), eosTokenIds.size()); stopWords = std::list(eosTokenIds.size()); - std::transform(eosTokenIds.cbegin(), eosTokenIds.cend(), stopWords.begin(), - [](const auto tokenIdObj) -> decltype(stopWords)::value_type { - const auto tokenId = tokenIdObj.template get(); - return {tokenId}; - }); + const auto to_single_token = [](const auto tokenIdObj) -> decltype(stopWords)::value_type { + return {tokenIdObj.template get()}; + }; + std::transform(eosTokenIds.cbegin(), eosTokenIds.cend(), stopWords.begin(), to_single_token); } } else { SPDLOG_INFO("No EOS tokens found, generation_config.json doesn't exist"); @@ -114,13 +113,13 @@ huggingface::tgi::backends::TensorRtLlmBackend::TensorRtLlmBackend( [[nodiscard("Returned number of requests needs to be consumed")]] size_t huggingface::tgi::backends::TensorRtLlmBackend::NumResponsesReady() const { +#ifdef NDEBUG + return executor.getNumResponsesReady(); +#else const auto numResponses = executor.getNumResponsesReady(); - -#ifndef NDEBUG - if(numResponses > 0) SPDLOG_INFO(FMT_STRING("Num responses ready: {:d}"), numResponses); -#endif - + if (numResponses > 0) SPDLOG_INFO(FMT_STRING("Num responses ready: {:d}"), numResponses); return numResponses; +#endif } [[nodiscard("Returned request id needs to be provided back to gather generated tokens")]]