huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-24 16:32:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

gpt_bigcode could also go pageattn

Browse Source 
Signed-off-by: Wang, Yi A <yi.a.wang@intel.com>
This commit is contained in:
Wang, Yi A 2025-03-18 23:59:31 -07:00
parent 073f793976
commit 2cde30de24
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
backends/gaudi/server/text_generation_server/models/__init__.py
View File

@ -486,8 +486,6 @@ def get_model(
model_type = config_dict["model_type"]
if model_type == "gpt_bigcode":
return StarCoder(model_id=model_id, revision=revision, dtype=dtype)
kv_cache_dtype = dtype
if FLASH_ATTENTION:
@ -871,6 +869,8 @@ def get_model(
trust_remote_code=trust_remote_code,
)
adapt_transformers_to_gaudi()
if model_type == "gpt_bigcode":
return StarCoder(model_id=model_id, revision=revision, dtype=dtype)
if model_type == "bloom":
return BLOOM(
model_id=model_id,