mirror of
https://github.com/huggingface/text-generation-inference.git
synced 2025-10-09 06:55:24 +00:00
75 lines
2.1 KiB
Plaintext
75 lines
2.1 KiB
Plaintext
FROM nvidia/cuda:12.6.3-cudnn-devel-ubuntu24.04 AS deps
|
|
|
|
ARG llama_version=b4628
|
|
ARG llama_cuda_arch=75-real;80-real;86-real;89-real;90-real
|
|
ENV TGI_LLAMA_PKG_CUDA=cuda-${CUDA_VERSION%.*}
|
|
|
|
WORKDIR /opt/src
|
|
|
|
ENV DEBIAN_FRONTEND=noninteractive
|
|
RUN apt update && apt install -y \
|
|
clang \
|
|
cmake \
|
|
curl \
|
|
git \
|
|
python3-dev \
|
|
libssl-dev \
|
|
pkg-config \
|
|
tar
|
|
|
|
ADD https://github.com/ggerganov/llama.cpp/archive/refs/tags/${llama_version}.tar.gz /opt/src/
|
|
RUN tar -xzf ${llama_version}.tar.gz \
|
|
&& cd llama.cpp-${llama_version} \
|
|
&& cmake -B build \
|
|
-DCMAKE_INSTALL_PREFIX=/usr \
|
|
-DCMAKE_INSTALL_LIBDIR=/usr/lib \
|
|
-DCMAKE_C_COMPILER=clang \
|
|
-DCMAKE_CXX_COMPILER=clang++ \
|
|
-DCMAKE_CUDA_ARCHITECTURES=${llama_cuda_arch} \
|
|
-DGGML_CUDA=1 \
|
|
-DLLAMA_BUILD_COMMON=OFF \
|
|
-DLLAMA_BUILD_TESTS=OFF \
|
|
-DLLAMA_BUILD_EXAMPLES=OFF \
|
|
-DLLAMA_BUILD_SERVER=OFF \
|
|
&& cmake --build build --parallel --config Release \
|
|
&& cmake --install build
|
|
|
|
WORKDIR /app
|
|
COPY rust-toolchain.toml rust-toolchain.toml
|
|
RUN curl -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --default-toolchain none
|
|
ENV PATH="/root/.cargo/bin:$PATH"
|
|
RUN cargo install cargo-chef --locked
|
|
|
|
FROM deps AS planner
|
|
COPY . .
|
|
RUN cargo chef prepare --recipe-path recipe.json
|
|
|
|
FROM deps AS builder
|
|
COPY --from=planner /app/recipe.json recipe.json
|
|
RUN cargo chef cook \
|
|
--recipe-path recipe.json \
|
|
--profile release-opt \
|
|
--package text-generation-router-llamacpp
|
|
COPY . .
|
|
RUN cargo build \
|
|
--profile release-opt \
|
|
--package text-generation-router-llamacpp --frozen
|
|
|
|
FROM nvidia/cuda:12.6.3-cudnn-runtime-ubuntu24.04
|
|
|
|
RUN apt update && apt install -y \
|
|
python3-venv \
|
|
python3-pip
|
|
|
|
RUN python3 -m venv /venv
|
|
ENV PATH="/venv/bin:$PATH"
|
|
|
|
COPY backends/llamacpp/requirements.txt requirements.txt
|
|
RUN pip3 install --no-cache-dir -r requirements.txt
|
|
|
|
COPY --from=builder /usr/lib/libllama.so /usr/lib/
|
|
COPY --from=builder /usr/lib/libggml*.so /usr/lib/
|
|
COPY --from=builder /app/target/release-opt/text-generation-router-llamacpp /usr/bin/
|
|
|
|
ENTRYPOINT ["text-generation-router-llamacpp"]
|