From 2158aaa3d93610809aea19b2d0e74bf05ba8c5d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Danie=CC=88l=20de=20Kok?= <me@danieldk.eu>
Date: Thu, 13 Mar 2025 12:36:56 +0000
Subject: [PATCH] Make the Nix-based Docker container work on non-NixOS

On NixOS, the CUDA driver shim gets mounted on /run/opengl-driver,
where Nix packages expect the shim to be. However, on other
distributions, some FHS paths are mounted. This is a small change
to make the dynamic loader find the shim.
---
 flake.nix      | 8 ++++++--
 nix/docker.nix | 4 ++++
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/flake.nix b/flake.nix
index ac4331c2..7ddd3b92 100644
--- a/flake.nix
+++ b/flake.nix
@@ -176,11 +176,15 @@
             '';
           };
 
-          dockerImage = pkgs.callPackage nix/docker.nix {
+          # Use plain nixpkgs without overlays for dockerTools. dockerTools
+          # uses a Python package for computing the layers from the transitive
+          # closure. However, this needs a lot of rebuilds due to our overlay.
+
+          dockerImage = nixpkgs.legacyPackages.${system}.callPackage nix/docker.nix {
             text-generation-inference = default;
           };
 
-          dockerImageStreamed = pkgs.callPackage nix/docker.nix {
+          dockerImageStreamed = nixpkgs.legacyPackages.${system}.callPackage nix/docker.nix {
             text-generation-inference = default;
             stream = true;
           };
diff --git a/nix/docker.nix b/nix/docker.nix
index dc4c823d..a8fc256e 100644
--- a/nix/docker.nix
+++ b/nix/docker.nix
@@ -18,6 +18,10 @@ build {
     Env = [
       "HF_HOME=/data"
       "PORT=80"
+      # The CUDA container toolkit will mount the driver shim into the
+      # container. We just have to ensure that the dynamic loader finds
+      # the libraries.
+      "LD_LIBRARY_PATH=/usr/lib64"
     ];
 
   };