huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-11 20:34:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

misc(backend): indent

Browse Source
This commit is contained in:
Morgan Funtowicz 2024-12-13 15:38:04 +01:00
parent ab6591e759
commit 1640da7c08
1 changed files with 42 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
backends/trtllm/csrc/backend.hpp
View File

@ -159,6 +159,7 @@ namespace huggingface::tgi::backends::trtllm {
public: public:
backend_t(std::filesystem::path &engines_folder, std::filesystem::path &executor_worker_path); backend_t(std::filesystem::path &engines_folder, std::filesystem::path &executor_worker_path);
backend_t(std::filesystem::path &&engines_folder, std::filesystem::path &&executor_worker_path) backend_t(std::filesystem::path &&engines_folder, std::filesystem::path &&executor_worker_path)
: backend_t(engines_folder, executor_worker_path) {}; : backend_t(engines_folder, executor_worker_path) {};
@ -171,7 +172,8 @@ namespace huggingface::tgi::backends::trtllm {
*/ */
[[nodiscard("Discarded executor request_id needs to be assigned")]] [[nodiscard("Discarded executor request_id needs to be assigned")]]
std::expected<request_id_t, backend_error_t> std::expected<request_id_t, backend_error_t>
submit(std::span<const token_id_t> token_ids, generation_params_t generation_params, sampling_params_t sampling_params) noexcept; submit(std::span<const token_id_t> token_ids, generation_params_t generation_params,
sampling_params_t sampling_params) noexcept;
/** /**
* Query the number of tokens available across all in-flight generations * Query the number of tokens available across all in-flight generations
@ -198,21 +200,26 @@ namespace huggingface::tgi::backends::trtllm {
* Create a TensorRT-LLM executor from a workspace * Create a TensorRT-LLM executor from a workspace
*/ */
const auto executor_factory_initializer = [](const backend_workspace_t &workspace) -> tle::Executor { const auto executor_factory_initializer = [](const backend_workspace_t &workspace) -> tle::Executor {
return { workspace.engines_folder(), tensorrt_llm::executor::ModelType::kDECODER_ONLY, workspace.executor_config() }; return {workspace.engines_folder(), tensorrt_llm::executor::ModelType::kDECODER_ONLY,
workspace.executor_config()};
}; };
} }
/** /**
* Helper structures to define formatting strategies for various types in the backend * Helper structures to define formatting strategies for various types in the backend
*/ */
template <> struct fmt::formatter<huggingface::tgi::backends::trtllm::generation_params_t>: formatter<string_view> { template<>
auto format(huggingface::tgi::backends::trtllm::generation_params_t const& c, format_context& ctx) const -> format_context::iterator { struct fmt::formatter<huggingface::tgi::backends::trtllm::generation_params_t> : formatter<string_view> {
auto format(huggingface::tgi::backends::trtllm::generation_params_t const &c,
format_context &ctx) const -> format_context::iterator {
return fmt::format_to(ctx.out(), "generation_params_t{{ max_new_tokens={:d} }}", c.max_new_tokens); return fmt::format_to(ctx.out(), "generation_params_t{{ max_new_tokens={:d} }}", c.max_new_tokens);
} }
}; };
template <> struct fmt::formatter<huggingface::tgi::backends::trtllm::sampling_params_t>: formatter<string_view> { template<>
auto format(huggingface::tgi::backends::trtllm::sampling_params_t const& c, format_context& ctx) const -> format_context::iterator { struct fmt::formatter<huggingface::tgi::backends::trtllm::sampling_params_t> : formatter<string_view> {
auto format(huggingface::tgi::backends::trtllm::sampling_params_t const &c,
format_context &ctx) const -> format_context::iterator {
return fmt::format_to( return fmt::format_to(
ctx.out(), ctx.out(),
"sampling_params_t{{ top_k={:d}, top_p={:.3f}, repetition_penalty={:.3f}, frequency_penalty={:.3f}, temperature={:.3f}, seed={:d} }}", "sampling_params_t{{ top_k={:d}, top_p={:.3f}, repetition_penalty={:.3f}, frequency_penalty={:.3f}, temperature={:.3f}, seed={:d} }}",
@ -220,4 +227,5 @@ template <> struct fmt::formatter<huggingface::tgi::backends::trtllm::sampling_p
); );
} }
}; };
#endif #endif