huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-12 04:44:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixing radix with block_size > 1

Browse Source
This commit is contained in:
Nicolas Patry 2024-09-13 18:01:56 +02:00
parent 4fc01e243d
commit b043f56ed2
No known key found for this signature in database
GPG Key ID: 64AF4752B2967863
4 changed files with 368 additions and 343 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
backends/v3/src/radix.rs
View File

@ -1,10 +1,17 @@
use crate::block_allocator::{Allocator, BlockAllocation};
use slotmap::{DefaultKey, SlotMap};
use std::hash::{Hash, Hasher};
use std::{
collections::{BTreeSet, HashMap},
sync::Arc,
};
fn hash_(slice: &[u32]) -> u32 {
let mut s = std::hash::DefaultHasher::new();
slice.hash(&mut s);
s.finish() as u32
}
pub struct RadixAllocator {
allocation_id: u64,
@ -44,6 +51,10 @@ impl RadixAllocator {
// the free list if we cannot allocate enough blocks. This is only
// temporary, the trie needs to be able to report whether it can
// allocate the requested amount. Just not implemented yet.
tracing::debug!(
"Free blocks {} need {n_blocks_needed}",
self.free_blocks.len()
);
self.free_blocks.extend(
self.cache_blocks
.evict(n_blocks_needed - self.free_blocks.len()),
@ -94,10 +105,12 @@ impl Allocator for RadixAllocator {
match self.alloc_or_reclaim(suffix_blocks as usize) {
Some(suffix_blocks) => blocks.extend(suffix_blocks),
None => {
tracing::debug!("Cannot allocate {:?}", self.cache_blocks);
tracing::debug!("Found {prefix_len} prefix tokens need {suffix_blocks} suffix blocks for {tokens} tokens");
tracing::debug!("Block size {}", self.block_size);
self.cache_blocks
.decref(prefix_node)
.expect("Failed to decrement refcount");
tracing::debug!("Cannot allocate {:?}", self.cache_blocks);
return None;
}
}
@ -269,7 +282,9 @@ impl RadixTrie {
fn find_(&mut self, mut node_id: NodeId, key: &[u32], blocks: &mut Vec<u32>) -> NodeId {
let node = &self.nodes[node_id];
if let Some(&child_id) = node.children.get(&key[0]) {
if key.len() >= self.block_size {
let node_key = hash_(&key[..self.block_size]);
if let Some(&child_id) = node.children.get(&node_key) {
self.update_access_time(child_id);
let child = self.nodes.get(child_id).expect("Invalid child identifier");
let shared_prefix_len = shared_prefix(&child.key, key, self.block_size);
@ -281,6 +296,7 @@ impl RadixTrie {
node_id = self.find_(child_id, key, blocks);
}
}
}
node_id
}
@ -345,9 +361,11 @@ impl RadixTrie {
// evict n_blocks and return `None` if we can't. We are now needlessly
// evicting prefixes from the cache in such a case.
let mut evicted = Vec::new();
tracing::debug!("Evicting in search of {n_blocks}");
while let Some((last_access, node_id)) = self.leaves.pop_first() {
let blocks_needed = n_blocks - evicted.len();
let blocks_needed = n_blocks.saturating_sub(evicted.len());
tracing::debug!("Evicting node {node_id:?} ");
let node = self.nodes.get(node_id).expect("Leave does not exist");
assert_eq!(
@ -369,8 +387,11 @@ impl RadixTrie {
// the required number of blocks and leave the remaining blocks
// untouched.
let node = self.nodes.get_mut(node_id).expect("Leave does not exist");
node.key.truncate(node.blocks.len() - blocks_needed);
evicted.extend(node.blocks.split_off(node.blocks.len() - blocks_needed));
let truncate_blocks = node.blocks.len() - blocks_needed;
let truncate_tokens = truncate_blocks * self.block_size;
node.key.truncate(truncate_tokens);
evicted.extend(node.blocks.split_off(truncate_blocks));
self.leaves.insert((last_access, node_id));
break;
}
@ -453,14 +474,15 @@ impl RadixTrie {
.get_mut(node_id)
.expect("Node to-be split does not exist");
let mut parent_key = node.key.split_off(prefix_len);
let mut parent_blocks = node.blocks.split_off(prefix_len);
let prefix_blocks = prefix_len / self.block_size;
let mut parent_blocks = node.blocks.split_off(prefix_blocks);
// Move first part of the prefix to the parent. We swap to avoid
// an allocation + copy for both splits of the key/blocks.
std::mem::swap(&mut node.key, &mut parent_key);
std::mem::swap(&mut node.blocks, &mut parent_blocks);
let node_key = node.key[0];
let node_key = hash_(&node.key[..self.block_size]);
let grandparent_id = node.parent.expect("Node does not have a parent");
let parent_id = self.add_node(grandparent_id, parent_key, parent_blocks);
@ -485,7 +507,7 @@ impl RadixTrie {
) -> NodeId {
let key = key.into();
let blocks = blocks.into();
let first = key[0];
let first = hash_(&key[..self.block_size]);
let child = TrieNode::new(key, blocks, self.time, Some(parent_id));
let child_id = self.nodes.insert(child);
@ -518,7 +540,9 @@ impl RadixTrie {
);
let parent_id = node.parent.expect("Attempted to remove root node");
let parent = self.nodes.get_mut(parent_id).expect("Unknown parent node");
parent.children.remove(&node.key[0]);
let node_key = hash_(&node.key[..self.block_size]);
parent.children.remove(&node_key);
self.decref(parent_id)
.expect("Failed to decrease parent refcount");
node

196
integration-tests/models/__snapshots__/test_flash_llama_prefix/test_flash_llama_flashdecoding.json → integration-tests/models/__snapshots__/test_flash_llama_prefix_flashdecoding/test_flash_llama_flashdecoding.json
View File

@ -14,7 +14,7 @@
"usage": null
}
],
"created": 1726129776,
"created": 1726243286,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -40,7 +40,7 @@
"usage": null
}
],
"created": 1726129776,
"created": 1726243278,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -66,7 +66,7 @@
"usage": null
}
],
"created": 1726129779,
"created": 1726243277,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -92,7 +92,7 @@
"usage": null
}
],
"created": 1726129779,
"created": 1726243278,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -118,7 +118,7 @@
"usage": null
}
],
"created": 1726129771,
"created": 1726243283,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -144,7 +144,7 @@
"usage": null
}
],
"created": 1726129777,
"created": 1726243286,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -170,7 +170,7 @@
"usage": null
}
],
"created": 1726129777,
"created": 1726243277,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -196,7 +196,7 @@
"usage": null
}
],
"created": 1726129770,
"created": 1726243278,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -222,7 +222,7 @@
"usage": null
}
],
"created": 1726129776,
"created": 1726243277,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -248,7 +248,7 @@
"usage": null
}
],
"created": 1726129779,
"created": 1726243284,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -274,7 +274,7 @@
"usage": null
}
],
"created": 1726129779,
"created": 1726243283,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -300,7 +300,7 @@
"usage": null
}
],
"created": 1726129770,
"created": 1726243277,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -326,7 +326,7 @@
"usage": null
}
],
"created": 1726129776,
"created": 1726243284,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -352,7 +352,7 @@
"usage": null
}
],
"created": 1726129771,
"created": 1726243278,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -378,7 +378,7 @@
"usage": null
}
],
"created": 1726129770,
"created": 1726243278,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -404,7 +404,7 @@
"usage": null
}
],
"created": 1726129777,
"created": 1726243284,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -430,7 +430,7 @@
"usage": null
}
],
"created": 1726129773,
"created": 1726243277,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -456,7 +456,7 @@
"usage": null
}
],
"created": 1726129776,
"created": 1726243277,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -482,7 +482,7 @@
"usage": null
}
],
"created": 1726129776,
"created": 1726243283,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -508,7 +508,7 @@
"usage": null
}
],
"created": 1726129776,
"created": 1726243278,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -534,7 +534,7 @@
"usage": null
}
],
"created": 1726129777,
"created": 1726243283,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -560,7 +560,7 @@
"usage": null
}
],
"created": 1726129779,
"created": 1726243286,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -586,7 +586,7 @@
"usage": null
}
],
"created": 1726129776,
"created": 1726243278,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -612,7 +612,7 @@
"usage": null
}
],
"created": 1726129779,
"created": 1726243286,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -638,7 +638,7 @@
"usage": null
}
],
"created": 1726129777,
"created": 1726243278,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -664,7 +664,7 @@
"usage": null
}
],
"created": 1726129776,
"created": 1726243278,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -690,7 +690,7 @@
"usage": null
}
],
"created": 1726129776,
"created": 1726243278,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -716,7 +716,7 @@
"usage": null
}
],
"created": 1726129776,
"created": 1726243277,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -742,7 +742,7 @@
"usage": null
}
],
"created": 1726129779,
"created": 1726243277,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -768,7 +768,7 @@
"usage": null
}
],
"created": 1726129771,
"created": 1726243278,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -794,7 +794,7 @@
"usage": null
}
],
"created": 1726129770,
"created": 1726243277,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -820,7 +820,7 @@
"usage": null
}
],
"created": 1726129779,
"created": 1726243277,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -846,7 +846,7 @@
"usage": null
}
],
"created": 1726129770,
"created": 1726243283,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -872,7 +872,7 @@
"usage": null
}
],
"created": 1726129770,
"created": 1726243278,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -898,7 +898,7 @@
"usage": null
}
],
"created": 1726129771,
"created": 1726243278,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -924,7 +924,7 @@
"usage": null
}
],
"created": 1726129777,
"created": 1726243278,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -950,7 +950,7 @@
"usage": null
}
],
"created": 1726129770,
"created": 1726243284,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -976,7 +976,7 @@
"usage": null
}
],
"created": 1726129779,
"created": 1726243284,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -1002,7 +1002,7 @@
"usage": null
}
],
"created": 1726129779,
"created": 1726243286,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -1028,7 +1028,7 @@
"usage": null
}
],
"created": 1726129779,
"created": 1726243284,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -1054,7 +1054,7 @@
"usage": null
}
],
"created": 1726129770,
"created": 1726243277,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -1080,7 +1080,7 @@
"usage": null
}
],
"created": 1726129777,
"created": 1726243284,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -1106,7 +1106,7 @@
"usage": null
}
],
"created": 1726129770,
"created": 1726243284,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -1132,7 +1132,7 @@
"usage": null
}
],
"created": 1726129771,
"created": 1726243286,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -1158,7 +1158,7 @@
"usage": null
}
],
"created": 1726129779,
"created": 1726243284,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -1184,7 +1184,7 @@
"usage": null
}
],
"created": 1726129770,
"created": 1726243278,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -1210,7 +1210,7 @@
"usage": null
}
],
"created": 1726129776,
"created": 1726243277,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -1236,7 +1236,7 @@
"usage": null
}
],
"created": 1726129779,
"created": 1726243277,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -1262,7 +1262,7 @@
"usage": null
}
],
"created": 1726129779,
"created": 1726243283,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -1288,7 +1288,7 @@
"usage": null
}
],
"created": 1726129770,
"created": 1726243277,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -1314,7 +1314,7 @@
"usage": null
}
],
"created": 1726129771,
"created": 1726243278,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -1340,7 +1340,7 @@
"usage": null
}
],
"created": 1726129779,
"created": 1726243284,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -1366,7 +1366,7 @@
"usage": null
}
],
"created": 1726129776,
"created": 1726243286,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -1392,7 +1392,7 @@
"usage": null
}
],
"created": 1726129770,
"created": 1726243283,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -1418,7 +1418,7 @@
"usage": null
}
],
"created": 1726129770,
"created": 1726243284,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -1444,7 +1444,7 @@
"usage": null
}
],
"created": 1726129776,
"created": 1726243286,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -1470,7 +1470,7 @@
"usage": null
}
],
"created": 1726129777,
"created": 1726243278,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -1496,7 +1496,7 @@
"usage": null
}
],
"created": 1726129777,
"created": 1726243283,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -1522,7 +1522,7 @@
"usage": null
}
],
"created": 1726129770,
"created": 1726243286,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -1548,7 +1548,7 @@
"usage": null
}
],
"created": 1726129770,
"created": 1726243277,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -1574,7 +1574,7 @@
"usage": null
}
],
"created": 1726129779,
"created": 1726243286,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -1600,7 +1600,7 @@
"usage": null
}
],
"created": 1726129776,
"created": 1726243278,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -1626,7 +1626,7 @@
"usage": null
}
],
"created": 1726129779,
"created": 1726243283,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -1652,7 +1652,7 @@
"usage": null
}
],
"created": 1726129771,
"created": 1726243284,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -1678,7 +1678,7 @@
"usage": null
}
],
"created": 1726129771,
"created": 1726243278,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -1704,7 +1704,7 @@
"usage": null
}
],
"created": 1726129779,
"created": 1726243283,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -1730,7 +1730,7 @@
"usage": null
}
],
"created": 1726129770,
"created": 1726243278,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -1756,7 +1756,7 @@
"usage": null
}
],
"created": 1726129771,
"created": 1726243284,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -1782,7 +1782,7 @@
"usage": null
}
],
"created": 1726129779,
"created": 1726243278,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -1808,7 +1808,7 @@
"usage": null
}
],
"created": 1726129779,
"created": 1726243284,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -1834,7 +1834,7 @@
"usage": null
}
],
"created": 1726129777,
"created": 1726243278,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -1860,7 +1860,7 @@
"usage": null
}
],
"created": 1726129779,
"created": 1726243283,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -1886,7 +1886,7 @@
"usage": null
}
],
"created": 1726129777,
"created": 1726243278,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -1912,7 +1912,7 @@
"usage": null
}
],
"created": 1726129770,
"created": 1726243286,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -1938,7 +1938,7 @@
"usage": null
}
],
"created": 1726129779,
"created": 1726243277,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -1964,7 +1964,7 @@
"usage": null
}
],
"created": 1726129776,
"created": 1726243278,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -1990,7 +1990,7 @@
"usage": null
}
],
"created": 1726129779,
"created": 1726243278,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -2016,7 +2016,7 @@
"usage": null
}
],
"created": 1726129770,
"created": 1726243286,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -2042,7 +2042,7 @@
"usage": null
}
],
"created": 1726129777,
"created": 1726243283,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -2068,7 +2068,7 @@
"usage": null
}
],
"created": 1726129771,
"created": 1726243278,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -2094,7 +2094,7 @@
"usage": null
}
],
"created": 1726129776,
"created": 1726243277,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -2120,7 +2120,7 @@
"usage": null
}
],
"created": 1726129779,
"created": 1726243284,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -2146,7 +2146,7 @@
"usage": null
}
],
"created": 1726129777,
"created": 1726243278,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -2172,7 +2172,7 @@
"usage": null
}
],
"created": 1726129779,
"created": 1726243284,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -2198,7 +2198,7 @@
"usage": null
}
],
"created": 1726129770,
"created": 1726243278,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -2224,7 +2224,7 @@
"usage": null
}
],
"created": 1726129776,
"created": 1726243278,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -2250,7 +2250,7 @@
"usage": null
}
],
"created": 1726129776,
"created": 1726243284,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -2276,7 +2276,7 @@
"usage": null
}
],
"created": 1726129777,
"created": 1726243277,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -2302,7 +2302,7 @@
"usage": null
}
],
"created": 1726129771,
"created": 1726243284,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -2328,7 +2328,7 @@
"usage": null
}
],
"created": 1726129779,
"created": 1726243283,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -2354,7 +2354,7 @@
"usage": null
}
],
"created": 1726129770,
"created": 1726243284,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -2380,7 +2380,7 @@
"usage": null
}
],
"created": 1726129776,
"created": 1726243284,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -2406,7 +2406,7 @@
"usage": null
}
],
"created": 1726129779,
"created": 1726243277,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -2432,7 +2432,7 @@
"usage": null
}
],
"created": 1726129776,
"created": 1726243277,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -2458,7 +2458,7 @@
"usage": null
}
],
"created": 1726129776,
"created": 1726243286,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -2484,7 +2484,7 @@
"usage": null
}
],
"created": 1726129770,
"created": 1726243283,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -2510,7 +2510,7 @@
"usage": null
}
],
"created": 1726129776,
"created": 1726243277,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",
@ -2536,7 +2536,7 @@
"usage": null
}
],
"created": 1726129779,
"created": 1726243284,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"object": "chat.completion",

228
integration-tests/models/test_flash_llama_prefix.py
View File

File diff suppressed because one or more lines are too long

229
integration-tests/models/test_flash_llama_prefix_flashdecoding.py Normal file
View File

File diff suppressed because one or more lines are too long