diff --git a/Dockerfile_trtllm b/Dockerfile_trtllm
index 441f59c8..72de0ea6 100644
--- a/Dockerfile_trtllm
+++ b/Dockerfile_trtllm
@@ -1,5 +1,4 @@
 ARG cuda_arch_list="75-real;80-real;86-real;89-real;90-real"
-ARG ompi_version="4.1.7"
 ARG build_type=release
 ARG sccache_gha_enabled=no
 ARG actions_cache_url=""
@@ -36,19 +35,16 @@ ENV TENSORRT_INSTALL_PREFIX=/usr/local/tensorrt
 
 # Install OpenMPI
 FROM cuda-builder AS mpi-builder
-ARG ompi_version
-
-ENV OMPI_TARBALL_FILENAME="openmpi-$ompi_version.tar.bz2"
-ADD --checksum=sha256:54a33cb7ad81ff0976f15a6cc8003c3922f0f3d8ceed14e1813ef3603f22cd34 \
-    https://download.open-mpi.org/release/open-mpi/v4.1/$OMPI_TARBALL_FILENAME \
-    /opt/src/mpi/
-
 WORKDIR /opt/src/mpi
-RUN tar --strip-components=1 -xf $OMPI_TARBALL_FILENAME &&\
+
+ADD --checksum=sha256:54a33cb7ad81ff0976f15a6cc8003c3922f0f3d8ceed14e1813ef3603f22cd34 \
+    https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-4.1.7.tar.bz2 .
+
+RUN tar --strip-components=1 -xf openmpi-4.1.7.tar.bz2 &&\
     ./configure --prefix=/usr/local/mpi --with-cuda=/usr/local/cuda --with-slurm && \
     make -j all && \
     make install && \
-    rm -rf "/opt/src/$OMPI_TARBALL_FILENAME"
+    rm -rf "/opt/src/openmpi-4.1.7.tar.bz2"
 
 # Install TensorRT
 FROM cuda-builder AS trt-builder
@@ -97,6 +93,9 @@ COPY --from=trt-builder /usr/local/tensorrt /usr/local/tensorrt
 COPY --from=mpi-builder /usr/local/mpi /usr/local/mpi
 
 ENV RUSTC_WRAPPER=sccache
+ENV CMAKE_C_COMPILER_LAUNCHER=sccache
+ENV CMAKE_CXX_COMPILER_LAUNCHER=sccache
+ENV CMAKE_CUDA_COMPILER_LAUNCHER=sccache
 ENV CMAKE_INSTALL_PREFIX=$TGI_INSTALL_PREFIX
 RUN mkdir $TGI_INSTALL_PREFIX && mkdir "$TGI_INSTALL_PREFIX/include" && mkdir "$TGI_INSTALL_PREFIX/lib" && \
     cargo build --profile ${build_type} --package text-generation-backends-trtllm --bin text-generation-backends-trtllm && \
diff --git a/backends/trtllm/CMakeLists.txt b/backends/trtllm/CMakeLists.txt
index 29c08e0c..26af80be 100644
--- a/backends/trtllm/CMakeLists.txt
+++ b/backends/trtllm/CMakeLists.txt
@@ -1,17 +1,5 @@
 cmake_minimum_required(VERSION 3.20)
 
-find_program(CCACHE_EXECUTABLE sccache)
-if (NOT CCACHE_EXECUTABLE)
-    find_program(CCACHE_EXECUTABLE ccache)
-endif ()
-
-if (CCACHE_EXECUTABLE)
-    message(STATUS "Using compiler with caching: ${CCACHE_EXECUTABLE}")
-    set(CMAKE_C_COMPILER_LAUNCHER "${CCACHE_EXECUTABLE}")
-    set(CMAKE_CXX_COMPILER_LAUNCHER "${CCACHE_EXECUTABLE}")
-    set(CMAKE_CUDA_COMPILER_LAUNCHER "${CCACHE_EXECUTABLE}")
-endif ()
-
 if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.24.0")
     cmake_policy(SET CMP0135 NEW)
 endif ()
diff --git a/backends/trtllm/build.rs b/backends/trtllm/build.rs
index 70d52132..429b7533 100644
--- a/backends/trtllm/build.rs
+++ b/backends/trtllm/build.rs
@@ -138,10 +138,6 @@ fn build_backend(is_debug: bool, opt_level: &str, out_dir: &PathBuf) -> (PathBuf
 
     if let Some(wrapper) = option_env!("RUSTC_WRAPPER") {
         println!("cargo:warning=Using caching tool: {wrapper}");
-
-        env::set_var("CMAKE_C_COMPILER_LAUNCHER", wrapper);
-        env::set_var("CMAKE_CXX_COMPILER_LAUNCHER", wrapper);
-        env::set_var("CMAKE_CUDA_COMPILER_LAUNCHER", wrapper);
     }
 
     // Allow to override which Python to use ...