huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-11 04:14:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Cleanup

Browse Source 
Signed-off-by: Adrien Gallouët <angt@huggingface.co>
This commit is contained in:
Adrien Gallouët 2025-02-07 12:08:34 +00:00
parent 6bdb644f2c
commit 0702e0bfda
No known key found for this signature in database
1 changed files with 14 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
backends/llamacpp/src/backend.rs
View File

@ -91,7 +91,7 @@ pub enum LlamacppGGMLType {
// TODO: macro
impl LlamacppGGMLType {
fn to_ggml_type(&self) -> llamacpp::ggml_type {
fn to_ggml_type(self) -> llamacpp::ggml_type {
match self {
LlamacppGGMLType::F32 => llamacpp::GGML_TYPE_F32,
LlamacppGGMLType::F16 => llamacpp::GGML_TYPE_F16,
@ -342,19 +342,21 @@ impl LlamacppSampler {
error!("Failed to init sampler");
return None;
}
let top_k = unsafe { llamacpp::sampler_init_top_k(req.top_k) };
let top_p = unsafe { llamacpp::sampler_init_top_p(req.top_p, req.min_keep) };
let typical_p = unsafe { llamacpp::sampler_init_typical(req.typical_p, req.min_keep) };
let temp = unsafe { llamacpp::sampler_init_temp(req.temp) };
let penalties = unsafe {
llamacpp::sampler_init_penalties(
req.penalty_last_n,
req.penalty_repeat,
req.penalty_freq,
req.penalty_present,
let (top_k, top_p, typical_p, temp, penalties, dist) = unsafe {
(
llamacpp::sampler_init_top_k(req.top_k),
llamacpp::sampler_init_top_p(req.top_p, req.min_keep),
llamacpp::sampler_init_typical(req.typical_p, req.min_keep),
llamacpp::sampler_init_temp(req.temp),
llamacpp::sampler_init_penalties(
req.penalty_last_n,
req.penalty_repeat,
req.penalty_freq,
req.penalty_present,
),
llamacpp::sampler_init_dist(req.seed),
)
};
let dist = unsafe { llamacpp::sampler_init_dist(req.seed) };
let all = &[
("top_k", top_k),
("top_p", top_p),