huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-25 12:02:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

(ffi) use const for GetSamplingConfig

Browse Source
This commit is contained in:
Morgan Funtowicz 2024-08-01 07:49:37 +00:00 committed by Morgan Funtowicz
parent cea64e234f
commit 0cd7538a48
2 changed files with 21 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
backends/trtllm/include/backend.h
View File

@ -48,12 +48,12 @@ namespace huggingface::tgi::backends {
* @return
*/
tle::SamplingConfig GetSamplingConfig(
uint32_t topK,
float_t topP,
float_t temperature,
float_t repetition_penalty,
float_t frequency_penalty,
uint64_t seed
const uint32_t topK,
const float_t topP,
const float_t temperature,
const float_t repetition_penalty,
const float_t frequency_penalty,
const uint64_t seed
);
/**
@ -94,13 +94,14 @@ namespace huggingface::tgi::backends {
* @return Request id related to this generation for reference
*/
[[nodiscard]] RequestId Submit(
const std::vector<TokenId> &tokens,
int32_t topK,
float_t topP,
float_t temperature,
float_t repetition_penalty,
float_t frequency_penalty,
uint64_t seed
const std::vector <TokenId> &tokens,
const uint32_t maxNewTokens,
const int32_t topK,
const float_t topP,
const float_t temperature,
const float_t repetition_penalty,
const float_t frequency_penalty,
const uint64_t seed
);
/**
@ -108,7 +109,7 @@ namespace huggingface::tgi::backends {
* @param requestId The request id to poll the generation results
* @return
*/
std::vector<tle::Response> Poll(RequestId requestId);
std::vector <tle::Response> Poll(RequestId requestId);
/**
* Stop the underlying executor

12
backends/trtllm/lib/backend.cpp
View File

@ -55,12 +55,12 @@ tle::ExecutorConfig huggingface::tgi::backends::GetExecutorConfig(const json &co
}
tle::SamplingConfig huggingface::tgi::backends::GetSamplingConfig(
uint32_t topK,
float_t topP,
float_t temperature,
float_t repetition_penalty,
float_t frequency_penalty,
uint64_t seed) {
const uint32_t topK,
const float_t topP,
const float_t temperature,
const float_t repetition_penalty,
const float_t frequency_penalty,
const uint64_t seed) {
return tle::SamplingConfig(
1, // TGI only use a single beam
topK,