mirror of
https://github.com/huggingface/text-generation-inference.git
synced 2025-04-21 23:12:07 +00:00
43df056eee
* (backend) use parking_lot crate for RwLock fairness
# Conflicts:
# backends/trtllm/src/backend.rs
* (launcher) default new server::run parameters to false for now
* (chore) fmt ... why?
* (ffi) use const for GetSamplingConfig
* (server) expose new SchedulingError
* (trt)
* (build) setup ccache if available
* (ffi) add max_new_tokens parameters
* (backend) cleanup a bit
* (backend) expose PullNewTokens
* (ffi) cleanup again
* (ffi) add missing headers imports
* (ffi) add template specialization to catch and convert to Rust Result<T, tensorrt_llm::common::TllmException>
* (looper) new looper initial implementation
* (ffi) remove narrowing type warning
* (ffi) encode the provided user prompt within each request thread
* (misc) change scope identifiers
* (backend) implement the post_processor background thread
* (misc) missing Result types for Rust
* use blocking_recv in looper to consume awaiting_requests at max before pulling in a single step
* (server) forward auth_token to server::run
* (build) fetchcontent use archives instead of git
* (ffi) fix usage of wrong vector constructor making a capacity fill call
* (ffi) missing namespace for tle::Response
* (ffi) do not use reference capture in lambda as we are not capturing anything
* (backend) refactor & cleanup
* (Dockerfile.trtllm) delete for now
* (misc) simplify [make_]move_iterator by using c++20 type inference
* (misc) no need to move for uint32_t items
* (scheduler) rework submit/pull logic
* (post) impl postprocessing
* (misc) delete backend.rs
* (misc) rerun-if-changed all the cmake modules
* (misc) move to latest trtllm
* (fix): HOPPER_SM_MAJOR is 9 not 8
* (misc: build for sm_{75,80,86,89,90} by default
* (misc): build with trtllm 0.13.0
* (misc): increase verbosity of spdlog
* (fix): do not recreate the stateful hashmap at every it
* (misc): update dependency in trtllm dockerfile
* (misc): update dependency in trtllm dockerfile
* (misc): disable logging in release mode
* (misc): improve trtllm download script robustness
* (fix): ore fixes for Dockerfile
* misc(cuda): require 12.6
* chore(cmake): use correct policy for download_timestamp
* feat(looper): check engine and executorWorker paths exist before creating the backend
* chore(cmake): download timestamp should be before URL
* feat(looper): minor optimizations to avoid growing too much the containers
* chore(trtllm): move dockerfile to right place
* chore(trtllm): disable tokenizer parallelism by default
* chore(trtllm): fmt
* chore(trtllm): post-rebase commit
* chore(trtllm): remove unused method
* feat(trtllm): cache maxNumTokens to avoid calling JSON everytime
* misc(router): remove SchedulingError
* feat(trtllm): do not tokenize twice
* Revert "chore(trtllm): remove unused method"
This reverts commit 31747163
* chore(rebase): fix invalid references
* chore(router): add python dependency
* Lint.
* Fix bad rebase
---------
Co-authored-by: Nicolas Patry <patry.nicolas@protonmail.com>
76 lines
1.9 KiB
C++
76 lines
1.9 KiB
C++
//
|
|
// Created by mfuntowicz on 7/11/24.
|
|
//
|
|
|
|
#ifndef TGI_TRTLLM_BACKEND_FFI_H
|
|
#define TGI_TRTLLM_BACKEND_FFI_H
|
|
|
|
#include <cmath>
|
|
#include <cstddef>
|
|
#include <memory>
|
|
#include "backend.h"
|
|
|
|
namespace huggingface::tgi::backends {
|
|
class TensorRtLlmBackendImpl;
|
|
}
|
|
|
|
// Template to support returning error from TllmException back to Rust in a Result<>
|
|
#include <tensorrt_llm/common/tllmException.h>
|
|
|
|
namespace rust::behavior {
|
|
template<typename Try, typename Fail>
|
|
static void trycatch(Try &&func, Fail &&fail) noexcept try {
|
|
func();
|
|
} catch (tensorrt_llm::common::TllmException &e) {
|
|
fail(e.what());
|
|
}
|
|
}
|
|
|
|
#include "backends/trtllm/src/lib.rs.h"
|
|
|
|
namespace huggingface::tgi::backends {
|
|
|
|
class TensorRtLlmBackendImpl : public TensorRtLlmBackend {
|
|
public:
|
|
/***
|
|
*
|
|
* @param engineFolder
|
|
* @param executorWorker
|
|
*/
|
|
TensorRtLlmBackendImpl(const std::string_view &engineFolder, const std::string_view &executorWorker);
|
|
|
|
/***
|
|
*
|
|
* @param tokens
|
|
* @param maxNewTokens
|
|
* @param topK
|
|
* @param topP
|
|
* @param temperature
|
|
* @param repetition_penalty
|
|
* @param frequency_penalty
|
|
* @param seed
|
|
* @return
|
|
*/
|
|
[[nodiscard("returned request id should be used to refer to the request's generation result later on")]]
|
|
uint64_t
|
|
Submit(rust::Slice<const uint32_t> tokens, uint32_t maxNewTokens,
|
|
int32_t topK, float_t topP, float_t temperature,
|
|
float_t repetition_penalty, float_t frequency_penalty, uint64_t seed);
|
|
|
|
/***
|
|
*
|
|
* @return
|
|
*/
|
|
std::unique_ptr<std::vector<GenerationStep>> PullTokens();
|
|
};
|
|
|
|
/***
|
|
*
|
|
* @param engineFolder
|
|
* @return
|
|
*/
|
|
std::unique_ptr<TensorRtLlmBackendImpl> CreateTensorRtLlmBackend(rust::Str engineFolder, rust::Str executorWorker);
|
|
}
|
|
|
|
#endif //TGI_TRTLLM_BACKEND_FFI_H
|