huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-24 16:32:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: fix local loading for .bin models (#1419)

Browse Source
This commit is contained in:
OlivierDehaene 2024-01-09 15:21:00 +01:00 committed by Karol Damaszke
parent fc9173aa59
commit 118344b99d
2 changed files with 3 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
server/text_generation_server/cli.py
View File

@ -242,7 +242,7 @@ def download_weights(
if not extension == ".safetensors" or not auto_convert:
raise e
else:
elif (Path(model_id) / "adapter_config.json").exists():
# Try to load as a local PEFT model
try:
utils.download_and_unload_peft(

5
server/text_generation_server/utils/peft.py
View File

@ -10,8 +10,7 @@ from peft import AutoPeftModelForCausalLM, AutoPeftModelForSeq2SeqLM
def download_and_unload_peft(model_id, revision, trust_remote_code):
torch_dtype = torch.float16
logger.info("Peft model detected.")
logger.info("Loading the model it might take a while without feedback")
logger.info("Trying to load a Peft model. It might take a while without feedback")
try:
model = AutoPeftModelForCausalLM.from_pretrained(
model_id,
@ -28,7 +27,7 @@ def download_and_unload_peft(model_id, revision, trust_remote_code):
trust_remote_code=trust_remote_code,
low_cpu_mem_usage=True,
)
logger.info(f"Loaded.")
logger.info("Peft model detected.")
logger.info(f"Merging the lora weights.")
base_model_id = model.peft_config["default"].base_model_name_or_path