huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-26 12:32:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

More logs in the allocator.

Browse Source
This commit is contained in:
Nicolas Patry 2025-01-28 11:19:37 +01:00
parent 065aabb13d
commit 7ef8b89ee7
No known key found for this signature in database
GPG Key ID: D2920555C90F704C
1 changed files with 12 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
backends/v3/src/radix.rs
View File

@ -57,13 +57,18 @@ impl RadixAllocator {
// temporary, the trie needs to be able to report whether it can // temporary, the trie needs to be able to report whether it can
// allocate the requested amount. Just not implemented yet. // allocate the requested amount. Just not implemented yet.
tracing::debug!( tracing::debug!(
"Free blocks {} need {n_blocks_needed}", "Free blocks {} need {n_blocks_needed}",
self.free_blocks.len() self.free_blocks.len()
); );
self.free_blocks.extend( let free_blocks = self
self.cache_blocks .cache_blocks
.evict(n_blocks_needed - self.free_blocks.len()), .evict(n_blocks_needed - self.free_blocks.len());
tracing::debug!(
"Freed {} blocks: Now having {} free blocks",
free_blocks.len(),
free_blocks.len() + self.free_blocks.len()
); );
self.free_blocks.extend(free_blocks);
} }
if self.free_blocks.len() >= n_blocks_needed { if self.free_blocks.len() >= n_blocks_needed {
@ -106,6 +111,9 @@ impl Allocator for RadixAllocator {
let suffix_blocks = suffix_len.div_ceil(self.block_size); let suffix_blocks = suffix_len.div_ceil(self.block_size);
tracing::info!("Prefix {prefix_len} - Suffix {suffix_len}"); tracing::info!("Prefix {prefix_len} - Suffix {suffix_len}");
metrics::counter!("tgi_cache_hit", "allocator" => "radix")
.increment(prefix_len.try_into().expect("Can convert usize to u64"));
metrics::counter!("tgi_cache_total", "allocator" => "radix").increment(suffix_len.into());
match self.alloc_or_reclaim(suffix_blocks as usize) { match self.alloc_or_reclaim(suffix_blocks as usize) {
Some(suffix_blocks) => blocks.extend(suffix_blocks), Some(suffix_blocks) => blocks.extend(suffix_blocks),