mirror of
https://github.com/huggingface/text-generation-inference.git
synced 2025-04-19 13:52:07 +00:00
43df056eee
* (backend) use parking_lot crate for RwLock fairness
# Conflicts:
# backends/trtllm/src/backend.rs
* (launcher) default new server::run parameters to false for now
* (chore) fmt ... why?
* (ffi) use const for GetSamplingConfig
* (server) expose new SchedulingError
* (trt)
* (build) setup ccache if available
* (ffi) add max_new_tokens parameters
* (backend) cleanup a bit
* (backend) expose PullNewTokens
* (ffi) cleanup again
* (ffi) add missing headers imports
* (ffi) add template specialization to catch and convert to Rust Result<T, tensorrt_llm::common::TllmException>
* (looper) new looper initial implementation
* (ffi) remove narrowing type warning
* (ffi) encode the provided user prompt within each request thread
* (misc) change scope identifiers
* (backend) implement the post_processor background thread
* (misc) missing Result types for Rust
* use blocking_recv in looper to consume awaiting_requests at max before pulling in a single step
* (server) forward auth_token to server::run
* (build) fetchcontent use archives instead of git
* (ffi) fix usage of wrong vector constructor making a capacity fill call
* (ffi) missing namespace for tle::Response
* (ffi) do not use reference capture in lambda as we are not capturing anything
* (backend) refactor & cleanup
* (Dockerfile.trtllm) delete for now
* (misc) simplify [make_]move_iterator by using c++20 type inference
* (misc) no need to move for uint32_t items
* (scheduler) rework submit/pull logic
* (post) impl postprocessing
* (misc) delete backend.rs
* (misc) rerun-if-changed all the cmake modules
* (misc) move to latest trtllm
* (fix): HOPPER_SM_MAJOR is 9 not 8
* (misc: build for sm_{75,80,86,89,90} by default
* (misc): build with trtllm 0.13.0
* (misc): increase verbosity of spdlog
* (fix): do not recreate the stateful hashmap at every it
* (misc): update dependency in trtllm dockerfile
* (misc): update dependency in trtllm dockerfile
* (misc): disable logging in release mode
* (misc): improve trtllm download script robustness
* (fix): ore fixes for Dockerfile
* misc(cuda): require 12.6
* chore(cmake): use correct policy for download_timestamp
* feat(looper): check engine and executorWorker paths exist before creating the backend
* chore(cmake): download timestamp should be before URL
* feat(looper): minor optimizations to avoid growing too much the containers
* chore(trtllm): move dockerfile to right place
* chore(trtllm): disable tokenizer parallelism by default
* chore(trtllm): fmt
* chore(trtllm): post-rebase commit
* chore(trtllm): remove unused method
* feat(trtllm): cache maxNumTokens to avoid calling JSON everytime
* misc(router): remove SchedulingError
* feat(trtllm): do not tokenize twice
* Revert "chore(trtllm): remove unused method"
This reverts commit 31747163
* chore(rebase): fix invalid references
* chore(router): add python dependency
* Lint.
* Fix bad rebase
---------
Co-authored-by: Nicolas Patry <patry.nicolas@protonmail.com>
108 lines
3.5 KiB
Plaintext
108 lines
3.5 KiB
Plaintext
ARG CUDA_ARCH_LIST="75-real;80-real;86-real;89-real;90-real"
|
|
ARG OMPI_VERSION="4.1.6"
|
|
|
|
# Build dependencies resolver stage
|
|
FROM lukemathwalker/cargo-chef:latest AS chef
|
|
WORKDIR /usr/src/text-generation-inference/backends/trtllm
|
|
|
|
FROM chef AS planner
|
|
COPY . .
|
|
RUN cargo chef prepare --recipe-path recipe.json
|
|
|
|
# CUDA dependent dependencies resolver stage
|
|
FROM nvidia/cuda:12.6.1-cudnn-devel-ubuntu22.04 AS cuda-builder
|
|
|
|
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
|
|
--mount=type=cache,target=/var/lib/apt,sharing=locked \
|
|
apt update && apt install -y \
|
|
build-essential \
|
|
cmake \
|
|
curl \
|
|
gcc \
|
|
g++ \
|
|
git \
|
|
git-lfs \
|
|
libssl-dev \
|
|
ninja-build \
|
|
pkg-config \
|
|
python3 \
|
|
python3-dev \
|
|
python3-setuptools \
|
|
tar \
|
|
wget
|
|
|
|
ENV TGI_INSTALL_PREFIX=/usr/local/tgi
|
|
ENV TENSORRT_INSTALL_PREFIX=/usr/local/tensorrt
|
|
|
|
# Install OpenMPI
|
|
FROM cuda-builder AS mpi-builder
|
|
ARG OMPI_VERSION
|
|
|
|
ENV OMPI_TARBALL_FILENAME="openmpi-$OMPI_VERSION.tar.bz2"
|
|
RUN wget "https://download.open-mpi.org/release/open-mpi/v4.1/$OMPI_TARBALL_FILENAME" -P /opt/src && \
|
|
mkdir /usr/src/mpi && \
|
|
tar -xf "/opt/src/$OMPI_TARBALL_FILENAME" -C /usr/src/mpi --strip-components=1 && \
|
|
cd /usr/src/mpi && \
|
|
./configure --prefix=/usr/local/mpi --with-cuda=/usr/local/cuda && \
|
|
make -j all && \
|
|
make install && \
|
|
rm -rf "/opt/src/$OMPI_TARBALL_FILENAME"
|
|
|
|
# Install TensorRT
|
|
FROM cuda-builder AS trt-builder
|
|
COPY backends/trtllm/scripts/install_tensorrt.sh /opt/install_tensorrt.sh
|
|
RUN chmod +x /opt/install_tensorrt.sh && \
|
|
/opt/install_tensorrt.sh
|
|
|
|
# Build Backend
|
|
FROM cuda-builder AS tgi-builder
|
|
WORKDIR /usr/src/text-generation-inference
|
|
|
|
# Install Rust
|
|
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | bash -s -- -y && \
|
|
chmod -R a+w /root/.rustup && \
|
|
chmod -R a+w /root/.cargo
|
|
|
|
ENV PATH="/root/.cargo/bin:$PATH"
|
|
RUN cargo install cargo-chef
|
|
|
|
# Cache dependencies
|
|
COPY --from=planner /usr/src/text-generation-inference/backends/trtllm/recipe.json .
|
|
RUN cargo chef cook --release --recipe-path recipe.json
|
|
|
|
# Build actual TGI
|
|
ARG CUDA_ARCH_LIST
|
|
ENV CMAKE_PREFIX_PATH="/usr/local/mpi:/usr/local/tensorrt:$CMAKE_PREFIX_PATH"
|
|
ENV LD_LIBRARY_PATH="/usr/local/mpi/lib:$LD_LIBRARY_PATH"
|
|
ENV PKG_CONFIG_PATH="/usr/local/mpi/lib/pkgconfig:$PKG_CONFIG_PATH"
|
|
|
|
COPY . .
|
|
COPY --from=trt-builder /usr/local/tensorrt /usr/local/tensorrt
|
|
COPY --from=mpi-builder /usr/local/mpi /usr/local/mpi
|
|
RUN mkdir $TGI_INSTALL_PREFIX && mkdir "$TGI_INSTALL_PREFIX/include" && mkdir "$TGI_INSTALL_PREFIX/lib" && \
|
|
cd backends/trtllm && \
|
|
CMAKE_INSTALL_PREFIX=$TGI_INSTALL_PREFIX cargo build --release
|
|
|
|
FROM nvidia/cuda:12.6.1-cudnn-runtime-ubuntu22.04 AS runtime
|
|
RUN apt update && apt install -y python3 && \
|
|
rm -rf /var/lib/{apt,dpkg,cache,log}/
|
|
|
|
WORKDIR /usr/local/tgi/bin
|
|
|
|
ENV LD_LIBRARY_PATH="/usr/local/tgi/lib:/usr/local/tensorrt/lib:/usr/local/cuda/lib64/stubs:$LD_LIBRARY_PATH"
|
|
ENV TOKENIZERS_PARALLELISM=false
|
|
ENV OMPI_MCA_plm_rsh_agent=""
|
|
|
|
COPY --from=mpi-builder /usr/local/mpi /usr/local/mpi
|
|
COPY --from=trt-builder /usr/local/tensorrt /usr/local/tensorrt
|
|
COPY --from=tgi-builder /usr/local/tgi /usr/local/tgi
|
|
COPY --from=tgi-builder /usr/src/text-generation-inference/target/release/text-generation-backends-trtllm /usr/local/tgi/bin/text-generation-launcher
|
|
|
|
FROM runtime
|
|
|
|
LABEL co.huggingface.vendor="Hugging Face Inc."
|
|
LABEL org.opencontainers.image.authors="hardware@hf.co"
|
|
|
|
ENTRYPOINT ["./text-generation-launcher"]
|
|
CMD ["--executor-worker", "/usr/local/tgi/bin/executorWorker"]
|