mirror of
https://github.com/huggingface/text-generation-inference.git
synced 2025-04-24 08:22:07 +00:00
abd58ff82c
# What does this PR do? Reworked the loading logic. Idea is to use cleaner loading code: - Remove need for `no_init_weights` - Remove all weird `bnb_linear` and `load_weights` and `post_load_weights`. New code layout: - New class `Weights` in charge of handling loading the weights from multiple files into appropiate tensors (potentially sharded) - TP layers now are "shells", they contain the code to know what kind of sharding we need + eventual `all_reduce`. They do not inherit from linear, but they contain some kind of Linear instead - the contained linear can be either FastLinear, BnbLinear or GPTq Linear next. - All modeling code is explictly made for sharding, process group is just no-ops for non sharded code (removes a lot of test cases)  --------- Co-authored-by: Ubuntu <ubuntu@ip-172-31-41-161.taildb5d.ts.net> Co-authored-by: Ubuntu <ubuntu@ip-172-31-41-161.ec2.internal> Co-authored-by: OlivierDehaene <olivier@huggingface.co> Co-authored-by: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com>
29 lines
1.1 KiB
Makefile
29 lines
1.1 KiB
Makefile
include Makefile-flash-att
|
|
|
|
unit-tests:
|
|
pytest -s -vv -m "not private" tests
|
|
|
|
gen-server:
|
|
# Compile protos
|
|
pip install grpcio-tools==1.51.1 mypy-protobuf==3.4.0 'types-protobuf>=3.20.4' --no-cache-dir
|
|
mkdir text_generation_server/pb || true
|
|
python -m grpc_tools.protoc -I../proto --python_out=text_generation_server/pb \
|
|
--grpc_python_out=text_generation_server/pb --mypy_out=text_generation_server/pb ../proto/generate.proto
|
|
find text_generation_server/pb/ -type f -name "*.py" -print0 -exec sed -i -e 's/^\(import.*pb2\)/from . \1/g' {} \;
|
|
touch text_generation_server/pb/__init__.py
|
|
|
|
install-torch:
|
|
# Install specific version of torch
|
|
pip install torch --extra-index-url https://download.pytorch.org/whl/cu118 --no-cache-dir
|
|
|
|
install: gen-server install-torch
|
|
pip install pip --upgrade
|
|
pip install -r requirements.txt
|
|
pip install -e ".[bnb, accelerate]"
|
|
|
|
run-dev:
|
|
SAFETENSORS_FAST_GPU=1 python -m torch.distributed.run --nproc_per_node=2 text_generation_server/cli.py serve bigscience/bloom-560m --sharded
|
|
|
|
export-requirements:
|
|
poetry export -o requirements.txt -E bnb --without-hashes
|