huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-27 13:02:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

update launcher

Browse Source
This commit is contained in:
Mohit Sharma 2024-06-24 15:09:17 +00:00
parent 3cc2f4e9fa
commit e81c4cf863
3 changed files with 4 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Dockerfile_amd
View File

@ -213,5 +213,5 @@ FROM base-copy
COPY ./tgi-entrypoint.sh /tgi-entrypoint.sh COPY ./tgi-entrypoint.sh /tgi-entrypoint.sh
RUN chmod +x /tgi-entrypoint.sh RUN chmod +x /tgi-entrypoint.sh
# ENTRYPOINT ["/tgi-entrypoint.sh"] ENTRYPOINT ["/tgi-entrypoint.sh"]
# CMD ["--json-output"] CMD ["--json-output"]

3
Makefile
View File

@ -53,6 +53,3 @@ run-falcon-7b-instruct-quantize:
clean: clean:
rm -rf target aml rm -rf target aml
interact:
docker run --rm -it --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --device=/dev/kfd --device=/dev/dri --group-add video --ipc=host --shm-size 64g --net host -v /home/mohit/.cache/huggingface/hub/:/data -v $(PWD):/tgi tgi-mht

4
launcher/src/main.rs
View File

@ -158,10 +158,10 @@ impl std::fmt::Display for KvDtype {
match self { match self {
KvDtype::Fp8 => { KvDtype::Fp8 => {
write!(f, "fp8") write!(f, "fp8")
}, }
KvDtype::Fp8e5m2 => { KvDtype::Fp8e5m2 => {
write!(f, "fp8_e5m2") write!(f, "fp8_e5m2")
}, }
} }
} }
} }