huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-11 12:24:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

Ensure all samplers are freed on error

Browse Source 
Signed-off-by: Adrien Gallouët <angt@huggingface.co>
This commit is contained in:
Adrien Gallouët 2025-02-05 15:42:59 +00:00
parent 5b777877b1
commit 695b1292e9
No known key found for this signature in database
1 changed files with 10 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
backends/llamacpp/src/backend.rs
View File

@ -365,14 +365,17 @@ impl LlamacppSampler {
let dist = unsafe { let dist = unsafe {
llamacpp::sampler_init_dist(req.seed) llamacpp::sampler_init_dist(req.seed)
}; };
let all = &[
("top_k", top_k),
("top_p", top_p),
("typical_p", typical_p),
("temp", temp),
("penalties", penalties),
("dist", dist),
];
let mut failed = false; let mut failed = false;
for (k, v) in &[( "top_k", top_k ), for (k, v) in all {
( "top_p", top_p ),
("typical_p", typical_p),
( "temp", temp ),
("penalties", penalties),
( "dist", dist )] {
if v.is_null() { if v.is_null() {
error!("Failed to init {k} sampler"); error!("Failed to init {k} sampler");
failed = true; failed = true;
@ -381,6 +384,7 @@ impl LlamacppSampler {
} }
} }
if failed { if failed {
unsafe { llamacpp::sampler_free(chain) };
None None
} else { } else {
Some(LlamacppSampler{chain}) Some(LlamacppSampler{chain})