huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-19 22:02:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix lora failure in platform which does not contain punica_kernels

Browse Source 
Signed-off-by: Wang, Yi A <yi.a.wang@intel.com>
This commit is contained in:
Wang, Yi A 2024-11-28 00:14:22 -08:00
parent caff779dd4
commit 98d0093660
1 changed files with 17 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
server/text_generation_server/adapters/lora.py
View File

@ -24,6 +24,7 @@ from text_generation_server.utils.sgmv import (
orient_for_rank, orient_for_rank,
pad_rank, pad_rank,
use_cutlass_shrink, use_cutlass_shrink,
has_sgmv,
) )
@ -325,6 +326,22 @@ class BatchLoraWeights(BatchAdapterWeights):
default=0, default=0,
) )
adapter_index_configs = {
idx: adapter_weights[idx].adapter_config
for idx in segment_indices
if idx in adapter_weights
}
use_sgmv = False
rank_data = {}
if not has_sgmv():
return BatchLoraWeights(
lora_a=lora_a,
lora_b=lora_b,
adapter_index_configs=adapter_index_configs,
rank_data=rank_data,
use_sgmv=use_sgmv,
)
if prefill or max_rank > BGMV_MAX_RANK: if prefill or max_rank > BGMV_MAX_RANK:
use_sgmv = True use_sgmv = True
lora_a_ptr = torch.tensor( lora_a_ptr = torch.tensor(
@ -378,12 +395,6 @@ class BatchLoraWeights(BatchAdapterWeights):
device=device, device=device,
) )
adapter_index_configs = {
idx: adapter_weights[idx].adapter_config
for idx in segment_indices
if idx in adapter_weights
}
adapter_to_segment = {v: k for k, v in enumerate(segment_indices)} adapter_to_segment = {v: k for k, v in enumerate(segment_indices)}
rank_indices = defaultdict(list) rank_indices = defaultdict(list)