huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-11 12:24:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

Ensure all samplers are freed on error

Browse Source 
Signed-off-by: Adrien Gallouët <angt@huggingface.co>
This commit is contained in:
Adrien Gallouët 2025-02-05 15:42:59 +00:00
parent 5b777877b1
commit 695b1292e9
No known key found for this signature in database
1 changed files with 10 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
backends/llamacpp/src/backend.rs
View File

@ -365,14 +365,17 @@ impl LlamacppSampler {
let dist = unsafe {
llamacpp::sampler_init_dist(req.seed)
};
let mut failed = false;
for (k, v) in &[( "top_k", top_k ),
let all = &[
("top_k", top_k),
("top_p", top_p),
("typical_p", typical_p),
("temp", temp),
("penalties", penalties),
( "dist", dist )] {
("dist", dist),
];
let mut failed = false;
for (k, v) in all {
if v.is_null() {
error!("Failed to init {k} sampler");
failed = true;
@ -381,6 +384,7 @@ impl LlamacppSampler {
}
}
if failed {
unsafe { llamacpp::sampler_free(chain) };
None
} else {
Some(LlamacppSampler{chain})