huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-11 04:14:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Cleanup

Browse Source 
Signed-off-by: Adrien Gallouët <angt@huggingface.co>
This commit is contained in:
Adrien Gallouët 2025-02-07 12:08:34 +00:00
parent 6bdb644f2c
commit 0702e0bfda
No known key found for this signature in database
1 changed files with 14 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
backends/llamacpp/src/backend.rs
View File

@ -91,7 +91,7 @@ pub enum LlamacppGGMLType {
// TODO: macro // TODO: macro
impl LlamacppGGMLType { impl LlamacppGGMLType {
fn to_ggml_type(&self) -> llamacpp::ggml_type { fn to_ggml_type(self) -> llamacpp::ggml_type {
match self { match self {
LlamacppGGMLType::F32 => llamacpp::GGML_TYPE_F32, LlamacppGGMLType::F32 => llamacpp::GGML_TYPE_F32,
LlamacppGGMLType::F16 => llamacpp::GGML_TYPE_F16, LlamacppGGMLType::F16 => llamacpp::GGML_TYPE_F16,
@ -342,19 +342,21 @@ impl LlamacppSampler {
error!("Failed to init sampler"); error!("Failed to init sampler");
return None; return None;
} }
let top_k = unsafe { llamacpp::sampler_init_top_k(req.top_k) }; let (top_k, top_p, typical_p, temp, penalties, dist) = unsafe {
let top_p = unsafe { llamacpp::sampler_init_top_p(req.top_p, req.min_keep) }; (
let typical_p = unsafe { llamacpp::sampler_init_typical(req.typical_p, req.min_keep) }; llamacpp::sampler_init_top_k(req.top_k),
let temp = unsafe { llamacpp::sampler_init_temp(req.temp) }; llamacpp::sampler_init_top_p(req.top_p, req.min_keep),
let penalties = unsafe { llamacpp::sampler_init_typical(req.typical_p, req.min_keep),
llamacpp::sampler_init_temp(req.temp),
llamacpp::sampler_init_penalties( llamacpp::sampler_init_penalties(
req.penalty_last_n, req.penalty_last_n,
req.penalty_repeat, req.penalty_repeat,
req.penalty_freq, req.penalty_freq,
req.penalty_present, req.penalty_present,
),
llamacpp::sampler_init_dist(req.seed),
) )
}; };
let dist = unsafe { llamacpp::sampler_init_dist(req.seed) };
let all = &[ let all = &[
("top_k", top_k), ("top_k", top_k),
("top_p", top_p), ("top_p", top_p),