huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-21 14:52:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore(trtllm): minor refactoring

Browse Source
This commit is contained in:
Morgan Funtowicz 2024-10-21 23:40:20 +02:00
parent 60a08a283d
commit 848b8ad554
1 changed files with 9 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
backends/trtllm/lib/backend.cpp
View File

@ -100,11 +100,10 @@ huggingface::tgi::backends::TensorRtLlmBackend::TensorRtLlmBackend(
SPDLOG_INFO(FMT_STRING("Found {:d} EOS tokens"), eosTokenIds.size());
stopWords = std::list<decltype(stopWords)::value_type>(eosTokenIds.size());
std::transform(eosTokenIds.cbegin(), eosTokenIds.cend(), stopWords.begin(),
[](const auto tokenIdObj) -> decltype(stopWords)::value_type {
const auto tokenId = tokenIdObj.template get<tle::TokenIdType>();
return {tokenId};
});
const auto to_single_token = [](const auto tokenIdObj) -> decltype(stopWords)::value_type {
return {tokenIdObj.template get<tle::TokenIdType>()};
};
std::transform(eosTokenIds.cbegin(), eosTokenIds.cend(), stopWords.begin(), to_single_token);
}
} else {
SPDLOG_INFO("No EOS tokens found, generation_config.json doesn't exist");
@ -114,13 +113,13 @@ huggingface::tgi::backends::TensorRtLlmBackend::TensorRtLlmBackend(
[[nodiscard("Returned number of requests needs to be consumed")]]
size_t huggingface::tgi::backends::TensorRtLlmBackend::NumResponsesReady() const {
#ifdef NDEBUG
return executor.getNumResponsesReady();
#else
const auto numResponses = executor.getNumResponsesReady();
#ifndef NDEBUG
if (numResponses > 0) SPDLOG_INFO(FMT_STRING("Num responses ready: {:d}"), numResponses);
#endif
return numResponses;
#endif
}
[[nodiscard("Returned request id needs to be provided back to gather generated tokens")]]