huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-08-02 05:10:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(trtllm): add new finish reasons

Browse Source 
Add new finish reasons introduced in TensorRT-LLM v0.16.0.
This commit is contained in:
Tzu-Yu Lee 2025-05-11 03:27:59 +08:00
parent 58934c8b61
commit c458d21d07
2 changed files with 12 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
backends/trtllm/csrc/ffi.hpp
View File

@ -42,6 +42,10 @@ namespace huggingface::tgi::backends::trtllm {
return finish_reason_t::kEND_ID; return finish_reason_t::kEND_ID;
case tle::FinishReason::kLENGTH: case tle::FinishReason::kLENGTH:
return finish_reason_t::kLENGTH; return finish_reason_t::kLENGTH;
case tle::FinishReason::kTIMED_OUT:
return finish_reason_t::kTIMED_OUT;
case tle::FinishReason::kCANCELLED:
return finish_reason_t::kCANCELLED;
default: default:
std::unreachable(); std::unreachable();
} }

8
backends/trtllm/src/lib.rs
View File

@ -24,6 +24,14 @@ mod ffi {
/// The request finished because the maximum number of tokens was reached. /// The request finished because the maximum number of tokens was reached.
#[cxx_name = "kLENGTH"] #[cxx_name = "kLENGTH"]
MaxLength = 3u8, MaxLength = 3u8,
#[cxx_name = "kTIMED_OUT"]
/// The request finished because it got timed out (via the mAllotedTime parameter)
TimedOut = 4u8,
#[cxx_name = "kCANCELLED"]
/// The request was cancelled by calling cancelRequest.
Cancelled = 5u8,
} }
/// Struct used as shared type between rust and C++ to represent the result /// Struct used as shared type between rust and C++ to represent the result