diff --git a/Dockerfile b/Dockerfile
index b6c5b2ed..6af58fd7 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -240,5 +240,9 @@ ENTRYPOINT ["./entrypoint.sh"]
 # Final image
 FROM base
 
+# Update ldcache so that triton can find libcuda.so
+RUN echo "/usr/local/cuda-12.1/compat" >> /etc/ld.so.conf.d/nvidia.conf && \
+    echo "/usr/local/cuda-12.1/lib64" >> /etc/ld.so.conf.d/nvidia.conf && ldconfig
+
 ENTRYPOINT ["text-generation-launcher"]
 CMD ["--json-output"]