huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-06-19 15:52:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

Only export the latest logits

Browse Source 
Signed-off-by: Adrien Gallouët <angt@huggingface.co>
This commit is contained in:
Adrien Gallouët 2025-02-01 10:51:44 +00:00
parent 960c12bd6e
commit 161280f313
No known key found for this signature in database
1 changed files with 4 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
backends/llamacpp/src/backend.rs
View File

@ -474,19 +474,21 @@ impl LlamacppBackend {
continue;
},
};
let last_pos = request.input_ids.len() - 1;
for (pos, &token_id) in request.input_ids.iter().enumerate() {
llamacpp.batch_push(
token_id as bindings::llama_token,
pos as bindings::llama_pos,
seq_id as bindings::llama_seq_id,
true, // TODO
pos == last_pos, // check samplers
);
}
seqs.push(LlamacppSeq {
id: seq_id,
batch_pos: llamacpp.batch.n_tokens as usize - 1,
token: -1,
pos: request.input_ids.len() as _,
pos: last_pos as bindings::llama_pos + 1,
sampler: sampler,
text: String::with_capacity(1024),
n_new_tokens: 0,