huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-24 08:22:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(backend): fix invalid reference to context in release mode

Browse Source
This commit is contained in:
Morgan Funtowicz 2024-11-11 19:50:33 +01:00
parent 7e2890fe2c
commit 488ba93898
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
backends/llamacpp/csrc/backend.cpp
View File

@ -74,7 +74,7 @@ namespace huggingface::tgi::backends::llamacpp {
const auto latency = std::chrono::duration_cast<std::chrono::milliseconds>(end - start); const auto latency = std::chrono::duration_cast<std::chrono::milliseconds>(end - start);
SPDLOG_DEBUG(FMT_STRING("Successfully decoded {:d} token(s) in {}"), batch.n_tokens, latency); SPDLOG_DEBUG(FMT_STRING("Successfully decoded {:d} token(s) in {}"), batch.n_tokens, latency);
#else #else
const auto status = llama_decode(context, batch); const auto status = llama_decode(context_.get(), batch);
#endif #endif
batch.n_tokens = 0; batch.n_tokens = 0;
if (LLAMA_SUCCESS(status)) [[likely]] { if (LLAMA_SUCCESS(status)) [[likely]] {