mirror of
https://github.com/huggingface/text-generation-inference.git
synced 2025-04-24 16:32:12 +00:00
ed1cfde0d8
# What does this PR do? <!-- Congratulations! You've made it this far! You're not quite done yet though. Once merged, your PR is going to appear in the release notes with the title you set, so make sure it's a great title that fully reflects the extent of your awesome contribution. Then, please replace this with a description of the change and which issue is fixed (if applicable). Please also include relevant motivation and context. List any dependencies (if any) that are required for this change. Once you're done, someone will review your PR shortly (see the section "Who can review?" below to tag some potential reviewers). They may suggest changes to make the code even better. If no one reviewed your PR after a week has passed, don't hesitate to post a new comment @-mentioning the same persons---sometimes notifications get lost. --> <!-- Remove if not applicable --> Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. <!-- Your PR will be replied to more quickly if you can figure out the right person to tag with @ @OlivierDehaene OR @Narsil -->
218 lines
6.6 KiB
Plaintext
218 lines
6.6 KiB
Plaintext
# Rust builder
|
|
FROM lukemathwalker/cargo-chef:latest-rust-1.78 AS chef
|
|
WORKDIR /usr/src
|
|
|
|
ARG CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse
|
|
|
|
FROM chef as planner
|
|
COPY Cargo.toml Cargo.toml
|
|
COPY rust-toolchain.toml rust-toolchain.toml
|
|
COPY proto proto
|
|
COPY benchmark benchmark
|
|
COPY router router
|
|
COPY launcher launcher
|
|
RUN cargo chef prepare --recipe-path recipe.json
|
|
|
|
FROM chef AS builder
|
|
|
|
RUN PROTOC_ZIP=protoc-21.12-linux-x86_64.zip && \
|
|
curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v21.12/$PROTOC_ZIP && \
|
|
unzip -o $PROTOC_ZIP -d /usr/local bin/protoc && \
|
|
unzip -o $PROTOC_ZIP -d /usr/local 'include/*' && \
|
|
rm -f $PROTOC_ZIP
|
|
|
|
COPY --from=planner /usr/src/recipe.json recipe.json
|
|
RUN cargo chef cook --profile release-opt --recipe-path recipe.json
|
|
|
|
ARG GIT_SHA
|
|
ARG DOCKER_LABEL
|
|
|
|
COPY Cargo.toml Cargo.toml
|
|
COPY rust-toolchain.toml rust-toolchain.toml
|
|
COPY proto proto
|
|
COPY benchmark benchmark
|
|
COPY router router
|
|
COPY launcher launcher
|
|
RUN cargo build --profile release-opt
|
|
|
|
# Text Generation Inference base image for RoCm
|
|
FROM rocm/dev-ubuntu-22.04:6.1.1_hip_update as base
|
|
|
|
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
|
|
build-essential \
|
|
ca-certificates \
|
|
ccache \
|
|
curl \
|
|
git \
|
|
make \
|
|
libssl-dev \
|
|
g++ \
|
|
# Needed to build VLLM & flash.
|
|
rocthrust-dev \
|
|
hipsparse-dev \
|
|
hipblas-dev \
|
|
hipblaslt-dev \
|
|
rocblas-dev \
|
|
hiprand-dev \
|
|
rocrand-dev \
|
|
miopen-hip-dev \
|
|
hipfft-dev \
|
|
hipcub-dev \
|
|
hipsolver-dev \
|
|
rccl-dev \
|
|
cmake \
|
|
python3-dev && \
|
|
rm -rf /var/lib/apt/lists/*
|
|
|
|
# Keep in sync with `server/pyproject.toml
|
|
ARG MAMBA_VERSION=23.1.0-1
|
|
ARG PYTORCH_VERSION='2.3.0'
|
|
ARG ROCM_VERSION='6.0.2'
|
|
ARG PYTHON_VERSION='3.10.10'
|
|
# Automatically set by buildx
|
|
ARG TARGETPLATFORM
|
|
ENV PATH /opt/conda/bin:$PATH
|
|
|
|
# TGI seem to require libssl.so.1.1 instead of libssl.so.3 so we can't use ubuntu 22.04. Ubuntu 20.04 has python==3.8, and TGI requires python>=3.9, hence the need for miniconda.
|
|
# Install mamba
|
|
# translating Docker's TARGETPLATFORM into mamba arches
|
|
RUN case ${TARGETPLATFORM} in \
|
|
"linux/arm64") MAMBA_ARCH=aarch64 ;; \
|
|
*) MAMBA_ARCH=x86_64 ;; \
|
|
esac && \
|
|
curl -fsSL -v -o ~/mambaforge.sh -O "https://github.com/conda-forge/miniforge/releases/download/${MAMBA_VERSION}/Mambaforge-${MAMBA_VERSION}-Linux-${MAMBA_ARCH}.sh"
|
|
RUN chmod +x ~/mambaforge.sh && \
|
|
bash ~/mambaforge.sh -b -p /opt/conda && \
|
|
mamba init && \
|
|
rm ~/mambaforge.sh
|
|
|
|
# Install flash-attention, torch dependencies
|
|
RUN pip install numpy einops ninja --no-cache-dir
|
|
|
|
RUN conda install intel::mkl-static intel::mkl-include
|
|
RUN pip uninstall -y triton && \
|
|
git clone --depth 1 --single-branch https://github.com/ROCm/triton.git && \
|
|
cd triton/python && \
|
|
pip install .
|
|
|
|
RUN git clone --depth 1 --recursive --single-branch --branch 2.3-patched https://github.com/fxmarty/pytorch.git pytorch && cd pytorch && pip install -r requirements.txt --no-cache-dir
|
|
|
|
ARG _GLIBCXX_USE_CXX11_ABI="1"
|
|
ARG CMAKE_PREFIX_PATH="/opt/conda"
|
|
ARG PYTORCH_ROCM_ARCH="gfx90a;gfx942"
|
|
ARG BUILD_CAFFE2="0" \
|
|
BUILD_CAFFE2_OPS="0" \
|
|
USE_CUDA="0" \
|
|
USE_ROCM="1" \
|
|
BUILD_TEST="0" \
|
|
USE_FBGEMM="0" \
|
|
USE_NNPACK="0" \
|
|
USE_QNNPACK="0" \
|
|
USE_XNNPACK="0" \
|
|
USE_FLASH_ATTENTION="1" \
|
|
USE_MEM_EFF_ATTENTION="0"
|
|
|
|
RUN cd pytorch && python tools/amd_build/build_amd.py && python setup.py install
|
|
|
|
# Set as recommended: https://github.com/ROCm/triton/wiki/A-script-to-set-program-execution-environment-in-ROCm
|
|
ENV HIP_FORCE_DEV_KERNARG=1
|
|
|
|
# On MI250 and MI300, performances for flash with Triton FA are slightly better than CK.
|
|
# However, Triton requires a tunning for each prompt length, which is prohibitive.
|
|
ENV ROCM_USE_FLASH_ATTN_V2_TRITON=0
|
|
|
|
FROM base AS kernel-builder
|
|
|
|
# # Build vllm kernels
|
|
FROM kernel-builder AS vllm-builder
|
|
WORKDIR /usr/src
|
|
|
|
COPY server/Makefile-vllm Makefile
|
|
|
|
# Build specific version of vllm
|
|
RUN make build-vllm-rocm
|
|
|
|
# Build Flash Attention v2 kernels
|
|
FROM kernel-builder AS flash-att-v2-builder
|
|
WORKDIR /usr/src
|
|
|
|
COPY server/Makefile-flash-att-v2 Makefile
|
|
|
|
# Build specific version of flash attention v2
|
|
RUN make build-flash-attention-v2-rocm
|
|
|
|
# Build Transformers CUDA kernels (gpt-neox and bloom)
|
|
FROM kernel-builder as custom-kernels-builder
|
|
WORKDIR /usr/src
|
|
COPY server/custom_kernels/ .
|
|
RUN python setup.py build
|
|
|
|
# Build exllama kernels
|
|
FROM kernel-builder as exllama-kernels-builder
|
|
WORKDIR /usr/src
|
|
COPY server/exllama_kernels/ .
|
|
|
|
RUN python setup.py build
|
|
|
|
# Build exllama v2 kernels
|
|
FROM kernel-builder as exllamav2-kernels-builder
|
|
WORKDIR /usr/src
|
|
COPY server/exllamav2_kernels/ .
|
|
|
|
RUN python setup.py build
|
|
|
|
FROM base as base-copy
|
|
|
|
# Text Generation Inference base env
|
|
ENV HUGGINGFACE_HUB_CACHE=/data \
|
|
HF_HUB_ENABLE_HF_TRANSFER=1 \
|
|
PORT=80
|
|
|
|
# Copy builds artifacts from vllm builder
|
|
COPY --from=vllm-builder /usr/src/vllm/build/lib.linux-x86_64-cpython-310 /opt/conda/lib/python3.10/site-packages
|
|
|
|
# Copy build artifacts from flash attention v2 builder
|
|
COPY --from=flash-att-v2-builder /usr/src/flash-attention-v2/build/lib.linux-x86_64-cpython-310 /opt/conda/lib/python3.10/site-packages
|
|
|
|
# Copy build artifacts from custom kernels builder
|
|
COPY --from=custom-kernels-builder /usr/src/build/lib.linux-x86_64-cpython-310 /opt/conda/lib/python3.10/site-packages
|
|
|
|
# Copy build artifacts from exllama kernels builder
|
|
COPY --from=exllama-kernels-builder /usr/src/build/lib.linux-x86_64-cpython-310 /opt/conda/lib/python3.10/site-packages
|
|
|
|
# Copy build artifacts from exllamav2 kernels builder
|
|
COPY --from=exllamav2-kernels-builder /usr/src/build/lib.linux-x86_64-cpython-310 /opt/conda/lib/python3.10/site-packages
|
|
|
|
# Install server
|
|
COPY proto proto
|
|
COPY server server
|
|
COPY server/Makefile server/Makefile
|
|
RUN cd server && \
|
|
make gen-server && \
|
|
pip install -r requirements_rocm.txt && \
|
|
pip install ".[accelerate, peft, outlines]" --no-cache-dir
|
|
|
|
# Install benchmarker
|
|
COPY --from=builder /usr/src/target/release-opt/text-generation-benchmark /usr/local/bin/text-generation-benchmark
|
|
# Install router
|
|
COPY --from=builder /usr/src/target/release-opt/text-generation-router /usr/local/bin/text-generation-router
|
|
# Install launcher
|
|
COPY --from=builder /usr/src/target/release-opt/text-generation-launcher /usr/local/bin/text-generation-launcher
|
|
|
|
# AWS Sagemaker compatible image
|
|
FROM base as sagemaker
|
|
|
|
COPY sagemaker-entrypoint.sh entrypoint.sh
|
|
RUN chmod +x entrypoint.sh
|
|
|
|
ENTRYPOINT ["./entrypoint.sh"]
|
|
|
|
# Final image
|
|
FROM base-copy
|
|
|
|
COPY ./tgi-entrypoint.sh /tgi-entrypoint.sh
|
|
RUN chmod +x /tgi-entrypoint.sh
|
|
|
|
ENTRYPOINT ["/tgi-entrypoint.sh"]
|
|
CMD ["--json-output"]
|