mirror of
https://github.com/huggingface/text-generation-inference.git
synced 2025-04-19 13:52:07 +00:00
abd58ff82c
# What does this PR do? Reworked the loading logic. Idea is to use cleaner loading code: - Remove need for `no_init_weights` - Remove all weird `bnb_linear` and `load_weights` and `post_load_weights`. New code layout: - New class `Weights` in charge of handling loading the weights from multiple files into appropiate tensors (potentially sharded) - TP layers now are "shells", they contain the code to know what kind of sharding we need + eventual `all_reduce`. They do not inherit from linear, but they contain some kind of Linear instead - the contained linear can be either FastLinear, BnbLinear or GPTq Linear next. - All modeling code is explictly made for sharding, process group is just no-ops for non sharded code (removes a lot of test cases)  --------- Co-authored-by: Ubuntu <ubuntu@ip-172-31-41-161.taildb5d.ts.net> Co-authored-by: Ubuntu <ubuntu@ip-172-31-41-161.ec2.internal> Co-authored-by: OlivierDehaene <olivier@huggingface.co> Co-authored-by: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com>
59 lines
1.8 KiB
Makefile
59 lines
1.8 KiB
Makefile
install-server:
|
|
cd server && make install
|
|
|
|
install-custom-kernels:
|
|
if [ "$$BUILD_EXTENSIONS" == "True" ]; then cd server/custom_kernels && python setup.py install; else echo "Custom kernels are disabled, you need set to BUILD_EXTENSION environment variable to 'True' in order to build them. (Please read the docs, kernels might not work on all hardware)"; fi
|
|
|
|
install-integration-tests:
|
|
cd integration-tests && pip install -r requirements.txt
|
|
cd clients/python && pip install .
|
|
|
|
install-router:
|
|
cd router && cargo install --path .
|
|
|
|
install-launcher:
|
|
cd launcher && cargo install --path .
|
|
|
|
install-benchmark:
|
|
cd benchmark && cargo install --path .
|
|
|
|
install: install-server install-router install-launcher install-custom-kernels
|
|
|
|
server-dev:
|
|
cd server && make run-dev
|
|
|
|
router-dev:
|
|
cd router && cargo run -- --port 8080
|
|
|
|
rust-tests: install-router install-launcher
|
|
cargo test
|
|
|
|
integration-tests: install-integration-tests
|
|
pytest -s -vv -m "not private" integration-tests
|
|
|
|
update-integration-tests: install-integration-tests
|
|
pytest -s -vv --snapshot-update integration-tests
|
|
|
|
python-server-tests:
|
|
HF_HUB_ENABLE_HF_TRANSFER=1 pytest -s -vv -m "not private" server/tests
|
|
|
|
python-client-tests:
|
|
pytest clients/python/tests
|
|
|
|
python-tests: python-server-tests python-client-tests
|
|
|
|
run-bloom-560m:
|
|
text-generation-launcher --model-id bigscience/bloom-560m --num-shard 2 --port 8080
|
|
|
|
run-bloom-560m-quantize:
|
|
text-generation-launcher --model-id bigscience/bloom-560m --num-shard 2 --quantize --port 8080
|
|
|
|
download-bloom:
|
|
HF_HUB_ENABLE_HF_TRANSFER=1 text-generation-server download-weights bigscience/bloom
|
|
|
|
run-bloom:
|
|
text-generation-launcher --model-id bigscience/bloom --num-shard 8 --port 8080
|
|
|
|
run-bloom-quantize:
|
|
text-generation-launcher --model-id bigscience/bloom --num-shard 8 --quantize --port 8080
|