huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-10 11:54:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

improve error handling

Browse Source
This commit is contained in:
michaelfeil 2023-07-22 23:50:38 +02:00
parent 7338e0097f
commit 74c31ee890
2 changed files with 8 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
server/Makefile
View File

@ -21,7 +21,7 @@ install-torch:
install: gen-server install-torch
pip install pip --upgrade
pip install -r requirements.txt
pip install -e ".[bnb, accelerate]"
pip install -e ".[bnb, accelerate, ct2]"
run-dev:
SAFETENSORS_FAST_GPU=1 python -m torch.distributed.run --nproc_per_node=2 text_generation_server/cli.py serve bigscience/bloom-560m --sharded

10
server/text_generation_server/models/ct2_causal_lm.py
View File

@ -79,8 +79,8 @@ class CT2CausalLM(Model):
# Start CT2 - conversion
out_dir = Path(HUGGINGFACE_HUB_CACHE) / \
f"ct2models-{model_id.replace('/','--')}--{ct2_compute_type}"
if not os.path.exists(out_dir / "model.bin"):
ex = ""
try:
converter = ctranslate2.converters.TransformersConverter(
model_id,
@ -97,15 +97,17 @@ class CT2CausalLM(Model):
force=True,
)
except Exception as ex:
pass
if not os.path.exists(out_dir / "model.bin") or ex:
raise ValueError(
f"conversion with ctranslate2 for {model_id} failed : Error {ex}"
)
if not os.path.exists(out_dir / "model.bin"):
raise ValueError(
f"no ctranslate2 for {model_id} found after conversion in {out_dir}"
)
# Start CT2
self.ct2_model = ctranslate2.Generator(
out_dir, device=self.ct2_device, compute_type=ct2_compute_type
str(out_dir), device=self.ct2_device, compute_type=ct2_compute_type
)
class DummyModel(torch.nn.Module):