huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-29 22:12:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(backend): use latest trtllm main version to have g++ >= 13 compatibility

Browse Source
This commit is contained in:
Morgan Funtowicz 2024-12-02 00:06:24 +01:00
parent fd7e2b5bbd
commit 71e700a6ea
2 changed files with 13 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
Dockerfile_trtllm
View File

@ -1,5 +1,5 @@
ARG CUDA_ARCH_LIST="75-real;80-real;86-real;89-real;90-real" ARG CUDA_ARCH_LIST="75-real;80-real;86-real;89-real;90-real"
ARG OMPI_VERSION="4.1.6" ARG OMPI_VERSION="4.1.7rc1"
# Build dependencies resolver stage # Build dependencies resolver stage
FROM lukemathwalker/cargo-chef:latest AS chef FROM lukemathwalker/cargo-chef:latest AS chef
@ -10,7 +10,7 @@ COPY . .
RUN cargo chef prepare --recipe-path recipe.json RUN cargo chef prepare --recipe-path recipe.json
# CUDA dependent dependencies resolver stage # CUDA dependent dependencies resolver stage
FROM nvidia/cuda:12.6.1-cudnn-devel-ubuntu22.04 AS cuda-builder FROM nvidia/cuda:12.6.1-cudnn-devel-ubuntu24.04 AS cuda-builder
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
--mount=type=cache,target=/var/lib/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt,sharing=locked \
@ -23,13 +23,16 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
git \ git \
git-lfs \ git-lfs \
libssl-dev \ libssl-dev \
libucx-dev \
ninja-build \ ninja-build \
pkg-config \ pkg-config \
pipx \
python3 \ python3 \
python3-dev \ python3-dev \
python3-setuptools \ python3-setuptools \
tar \ tar \
wget wget && \
pipx ensurepath
ENV TGI_INSTALL_PREFIX=/usr/local/tgi ENV TGI_INSTALL_PREFIX=/usr/local/tgi
ENV TENSORRT_INSTALL_PREFIX=/usr/local/tensorrt ENV TENSORRT_INSTALL_PREFIX=/usr/local/tensorrt
@ -83,10 +86,11 @@ RUN mkdir $TGI_INSTALL_PREFIX && mkdir "$TGI_INSTALL_PREFIX/include" && mkdir "$
cd backends/trtllm && \ cd backends/trtllm && \
CMAKE_INSTALL_PREFIX=$TGI_INSTALL_PREFIX cargo build --release CMAKE_INSTALL_PREFIX=$TGI_INSTALL_PREFIX cargo build --release
FROM nvidia/cuda:12.6.1-cudnn-runtime-ubuntu22.04 AS runtime FROM nvidia/cuda:12.6.1-cudnn-runtime-ubuntu24.04 AS runtime
RUN apt update && apt install -y python3-minimal python3-dev python3-pip && \ RUN apt update && apt install -y libucx0 pipx python3-minimal python3-dev python3-pip python3-venv && \
rm -rf /var/lib/{apt,dpkg,cache,log}/ && \ rm -rf /var/lib/{apt,dpkg,cache,log}/ && \
python3 -m pip install transformers tokenizers pipx ensurepath && \
pipx install --include-deps transformers tokenizers
WORKDIR /usr/local/tgi/bin WORKDIR /usr/local/tgi/bin

5
backends/trtllm/cmake/trtllm.cmake
View File

@ -11,6 +11,7 @@ set(CMAKE_CUDA_ARCHITECTURES ${TGI_TRTLLM_BACKEND_TARGET_CUDA_ARCH_LIST})
message(STATUS "Building for CUDA Architectures: ${CMAKE_CUDA_ARCHITECTURES}") message(STATUS "Building for CUDA Architectures: ${CMAKE_CUDA_ARCHITECTURES}")
set(ENABLE_UCX OFF)
if (${CMAKE_BUILD_TYPE} STREQUAL "Debug") if (${CMAKE_BUILD_TYPE} STREQUAL "Debug")
set(FAST_BUILD ON) set(FAST_BUILD ON)
set(NVTX_DISABLE OFF) set(NVTX_DISABLE OFF)
@ -23,8 +24,8 @@ endif ()
fetchcontent_declare( fetchcontent_declare(
trtllm trtllm
GIT_REPOSITORY https://github.com/NVIDIA/TensorRT-LLM.git GIT_REPOSITORY https://github.com/NVIDIA/TensorRT-LLM.git
GIT_TAG 201135e58aa525af7e523d091d4c9584229524bc GIT_TAG c994b697313b8d36527ba346a0c5e8e374616c95
GIT_SHALLOW FALSE GIT_SHALLOW ON
DOWNLOAD_EXTRACT_TIMESTAMP DOWNLOAD_EXTRACT_TIMESTAMP
) )
fetchcontent_makeavailable(trtllm) fetchcontent_makeavailable(trtllm)