mirror of
https://github.com/huggingface/text-generation-inference.git
synced 2025-04-21 23:12:07 +00:00
04e1af94d7
* feat: first draft load multiple lora * feat: load weights within layer and refactor lora pass * fix: refactor and reduce lora math * feat: baseline impl single request multi lora support * feat: prefer lorax implementation and port loading logic * fix: prefer adapter_data and refactors * feat: perfer loraxs custom punica kernels and add mlp loras * fix: adjust batch for bgmv * fix: adjust adapter_segments logic when in batch * fix: refactor and move changes to v3 proto * fix: pass model_id for all flash causal lms * fix: pass model_id for all causal and seq2seq lms * fix: add model_id to model test * feat: add lora support to mistral and refactors * feat: prefer model id in request * fix: include rust code for adapter id * feat: bump launcher and add new lora docs * feat: support base model generation and refactors * fix: rename doc to retry ci build * feat: support if vlm models * fix: add adapter_data param and avoid missing layers * fix: add adapter_data param to phi and neox * fix: update all models forwards to include adapter_data * fix: add model_id to IdeficsCausalLM * Update lora.md Fixed a typo * Update lora.md Fixing spam image * fix: add lora kernel to dockerfile, support running without kernels and refactors * fix: avoid dockerfile conflict * fix: refactors and adjust flash llama lora logic * fix: skip llama test due to CI issue (temp) * fix: skip llama test CI (temp) 2 * fix: revert skips and prefer updated ci token for tests * fix: refactors and helpful comments * fix: add noop in TensorParallelAdapterRowLinear too * fix: refactor and move shard_lora_weights logic * fix: exit early if no adapter_data --------- Co-authored-by: Derek <datavistics@gmail.com>
40 lines
1.4 KiB
Makefile
40 lines
1.4 KiB
Makefile
include Makefile-flash-att
|
|
include Makefile-flash-att-v2
|
|
include Makefile-vllm
|
|
include Makefile-awq
|
|
include Makefile-eetq
|
|
include Makefile-selective-scan
|
|
include Makefile-lorax-punica
|
|
|
|
unit-tests:
|
|
pytest -s -vv -m "not private" tests
|
|
|
|
gen-server:
|
|
# Compile protos
|
|
pip install grpcio-tools==1.62.2 mypy-protobuf==3.6.0 'types-protobuf' --no-cache-dir
|
|
mkdir text_generation_server/pb || true
|
|
python -m grpc_tools.protoc -I../proto/v3 --python_out=text_generation_server/pb \
|
|
--grpc_python_out=text_generation_server/pb --mypy_out=text_generation_server/pb ../proto/v3/generate.proto
|
|
find text_generation_server/pb/ -type f -name "*.py" -print0 -exec sed -i -e 's/^\(import.*pb2\)/from . \1/g' {} \;
|
|
touch text_generation_server/pb/__init__.py
|
|
|
|
install-server: gen-server
|
|
pip install pip --upgrade
|
|
pip install -r requirements_cuda.txt
|
|
pip install -e ".[bnb, accelerate, quantize, peft, outlines]"
|
|
|
|
|
|
install: install-cuda
|
|
echo "Installed server"
|
|
|
|
install-cuda: install-server install-flash-attention-v2-cuda install-vllm-cuda install-flash-attention
|
|
|
|
install-rocm: install-server install-flash-attention-v2-rocm install-vllm-rocm
|
|
|
|
run-dev:
|
|
SAFETENSORS_FAST_GPU=1 python -m torch.distributed.run --nproc_per_node=2 text_generation_server/cli.py serve bigscience/bloom-560m --sharded
|
|
|
|
export-requirements:
|
|
poetry export -o requirements_cuda.txt --without-hashes
|
|
poetry export -o requirements_rocm.txt --without-hashes
|