mirror of
https://github.com/huggingface/text-generation-inference.git
synced 2025-04-25 03:52:08 +00:00
157 lines
5.5 KiB
Plaintext
157 lines
5.5 KiB
Plaintext
ARG cuda_arch_list="75-real;80-real;86-real;89-real;90-real"
|
|
ARG ompi_version="4.1.7rc1"
|
|
ARG build_type=release
|
|
ARG is_gha_build=false
|
|
|
|
# CUDA dependent dependencies resolver stage
|
|
FROM nvidia/cuda:12.6.3-cudnn-devel-ubuntu24.04 AS cuda-builder
|
|
|
|
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
|
|
build-essential \
|
|
cmake \
|
|
curl \
|
|
gcc-14 \
|
|
g++-14 \
|
|
git \
|
|
git-lfs \
|
|
lld \
|
|
libssl-dev \
|
|
libucx-dev \
|
|
libasan8 \
|
|
libubsan1 \
|
|
ninja-build \
|
|
pkg-config \
|
|
pipx \
|
|
python3 \
|
|
python3-dev \
|
|
python3-setuptools \
|
|
tar \
|
|
wget --no-install-recommends && \
|
|
pipx ensurepath
|
|
|
|
ENV TGI_INSTALL_PREFIX=/usr/local/tgi
|
|
ENV TENSORRT_INSTALL_PREFIX=/usr/local/tensorrt
|
|
|
|
# Install OpenMPI
|
|
FROM cuda-builder AS mpi-builder
|
|
ARG ompi_version
|
|
|
|
ENV OMPI_TARBALL_FILENAME="openmpi-$ompi_version.tar.bz2"
|
|
RUN wget "https://download.open-mpi.org/release/open-mpi/v4.1/$OMPI_TARBALL_FILENAME" -P /opt/src && \
|
|
mkdir /usr/src/mpi && \
|
|
tar -xf "/opt/src/$OMPI_TARBALL_FILENAME" -C /usr/src/mpi --strip-components=1 && \
|
|
cd /usr/src/mpi && \
|
|
./configure --prefix=/usr/local/mpi --with-cuda=/usr/local/cuda --with-slurm && \
|
|
make -j all && \
|
|
make install && \
|
|
rm -rf "/opt/src/$OMPI_TARBALL_FILENAME"
|
|
|
|
# Install TensorRT
|
|
FROM cuda-builder AS trt-builder
|
|
COPY backends/trtllm/scripts/install_tensorrt.sh /opt/install_tensorrt.sh
|
|
RUN chmod +x /opt/install_tensorrt.sh && \
|
|
/opt/install_tensorrt.sh
|
|
|
|
# Build Backend
|
|
FROM cuda-builder AS tgi-builder
|
|
WORKDIR /usr/src/text-generation-inference
|
|
|
|
# Scoped global args reuse
|
|
ARG is_gha_build
|
|
ARG build_type
|
|
|
|
# Install Rust
|
|
ENV PATH="/root/.cargo/bin:$PATH"
|
|
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | bash -s -- -y && \
|
|
chmod -R a+w /root/.rustup && \
|
|
chmod -R a+w /root/.cargo && \
|
|
cargo install sccache --locked
|
|
|
|
# SCCACHE Specifics args - before finding a better, more generic, way...
|
|
ARG aws_access_key_id
|
|
ARG aws_secret_access_key
|
|
ARG aws_session_token
|
|
ARG sccache_bucket
|
|
ARG sccache_s3_key_prefix
|
|
ARG sccache_region
|
|
|
|
ENV AWS_ACCESS_KEY_ID=$aws_access_key_id
|
|
ENV AWS_SECRET_ACCESS_KEY=$aws_secret_access_key
|
|
ENV AWS_SESSION_TOKEN=$aws_session_token
|
|
ENV SCCACHE_BUCKET=$sccache_bucket
|
|
ENV SCCACHE_S3_KEY_PREFIX=$sccache_s3_key_prefix
|
|
ENV SCCACHE_REGION=$sccache_region
|
|
|
|
ENV LD_LIBRARY_PATH="/usr/local/mpi/lib:$LD_LIBRARY_PATH"
|
|
ENV PKG_CONFIG_PATH="/usr/local/mpi/lib/pkgconfig:$PKG_CONFIG_PATH"
|
|
ENV CMAKE_PREFIX_PATH="/usr/local/mpi:/usr/local/tensorrt:$CMAKE_PREFIX_PATH"
|
|
|
|
ENV USE_LLD_LINKER=ON
|
|
ENV CUDA_ARCH_LIST=${cuda_arch_list}
|
|
ENV IS_GHA_BUILD=${is_gha_build}
|
|
|
|
COPY Cargo.lock Cargo.lock
|
|
COPY Cargo.toml Cargo.toml
|
|
COPY rust-toolchain.toml rust-toolchain.toml
|
|
COPY router router
|
|
COPY backends backends
|
|
COPY benchmark benchmark
|
|
COPY launcher launcher
|
|
COPY --from=trt-builder /usr/local/tensorrt /usr/local/tensorrt
|
|
COPY --from=mpi-builder /usr/local/mpi /usr/local/mpi
|
|
|
|
RUN mkdir $TGI_INSTALL_PREFIX && mkdir "$TGI_INSTALL_PREFIX/include" && mkdir "$TGI_INSTALL_PREFIX/lib" && \
|
|
python3 backends/trtllm/scripts/setup_sccache.py --is-gha-build ${is_gha_build} && \
|
|
CMAKE_INSTALL_PREFIX=$TGI_INSTALL_PREFIX \
|
|
RUSTC_WRAPPER=sccache \
|
|
cargo build --profile ${build_type} --package text-generation-backends-trtllm --bin text-generation-backends-trtllm && \
|
|
sccache --show-stats
|
|
|
|
FROM nvidia/cuda:12.6.3-cudnn-runtime-ubuntu24.04 AS runtime
|
|
RUN apt update && apt install -y libucx0 pipx python3-minimal python3-dev python3-pip python3-venv && \
|
|
rm -rf /var/lib/{apt,dpkg,cache,log}/ && \
|
|
pipx ensurepath && \
|
|
pipx install --include-deps transformers tokenizers
|
|
|
|
WORKDIR /usr/local/tgi/bin
|
|
|
|
ENV PATH=/root/.local/share/pipx/venvs/transformers/bin/:$PATH
|
|
ENV LD_LIBRARY_PATH="/usr/local/tgi/lib:/usr/local/mpi/lib:/usr/local/tensorrt/lib:/usr/local/cuda/lib64/stubs:$LD_LIBRARY_PATH"
|
|
ENV TOKENIZERS_PARALLELISM=false
|
|
ENV OMPI_MCA_plm_rsh_agent=""
|
|
|
|
COPY --from=mpi-builder /usr/local/mpi /usr/local/mpi
|
|
COPY --from=trt-builder /usr/local/tensorrt /usr/local/tensorrt
|
|
COPY --from=tgi-builder /usr/local/tgi /usr/local/tgi
|
|
COPY --from=tgi-builder /usr/src/text-generation-inference/target/release/text-generation-backends-trtllm /usr/local/tgi/bin/text-generation-launcher
|
|
|
|
FROM runtime
|
|
|
|
LABEL co.huggingface.vendor="Hugging Face Inc."
|
|
LABEL org.opencontainers.image.authors="hardware@hf.co"
|
|
LABEL org.opencontainers.title="Text-Generation-Inference TensorRT-LLM Backend"
|
|
|
|
ENTRYPOINT ["./text-generation-launcher"]
|
|
CMD ["--executor-worker", "/usr/local/tgi/bin/executorWorker"]
|
|
|
|
# This is used only for the CI/CD
|
|
FROM nvidia/cuda:12.6.3-cudnn-runtime-ubuntu24.04 AS ci-runtime
|
|
RUN apt update && apt install -y libasan8 libubsan1 libucx0 pipx python3-minimal python3-dev python3-pip python3-venv && \
|
|
rm -rf /var/lib/{apt,dpkg,cache,log}/ && \
|
|
pipx ensurepath && \
|
|
pipx install --include-deps transformers tokenizers
|
|
|
|
WORKDIR /usr/local/tgi/bin
|
|
|
|
ENV PATH=/root/.local/share/pipx/venvs/transformers/bin/:$PATH
|
|
ENV LD_LIBRARY_PATH="/usr/local/tgi/lib:/usr/local/mpi/lib:/usr/local/tensorrt/lib:/usr/local/cuda/lib64/stubs:$LD_LIBRARY_PATH"
|
|
ENV TOKENIZERS_PARALLELISM=false
|
|
ENV OMPI_MCA_plm_rsh_agent=""
|
|
|
|
COPY --from=mpi-builder /usr/local/mpi /usr/local/mpi
|
|
COPY --from=trt-builder /usr/local/tensorrt /usr/local/tensorrt
|
|
COPY --from=tgi-builder /usr/local/tgi /usr/local/tgi
|
|
|
|
# Basically we copy from target/debug instead of target/release
|
|
COPY --from=tgi-builder /usr/src/text-generation-inference/target/debug/text-generation-backends-trtllm /usr/local/tgi/bin/text-generation-launcher
|