huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-24 08:22:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Forgot last default place.

Browse Source
This commit is contained in:
Nicolas Patry 2024-08-20 18:17:54 +02:00
parent 860b550cdf
commit 8d0220a695
No known key found for this signature in database
GPG Key ID: 64AF4752B2967863
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
backends/v3/src/backend.rs
View File

@ -47,7 +47,7 @@ impl BackendV3 {
} else if prefix_caching {
Attention::FlashInfer
} else {
Attention::Paged
Attention::FlashDecoding
};
let block_size = if attention == Attention::FlashDecoding {
256