huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-27 13:02:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

nix: experimental support for building a Docker image

Browse Source 
Run using something like:

```
docker run \
  --device nvidia.com/gpu=all \
  -it --rm -p 8080:80 \
  -v $PWD/data:/data \
  -v $PWD/tmp:/tmp \
  tgi-docker:latest \
  --model-id <model_id>
```
This commit is contained in:
Daniël de Kok 2024-08-29 08:16:37 +00:00
parent 5b6b74e21d
commit 77a36d45eb
1 changed files with 25 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
flake.nix
View File

@ -141,15 +141,31 @@
}; };
}; };
packages.default = pkgs.writeShellApplication { packages = rec {
name = "text-generation-inference"; default = pkgs.writeShellApplication {
runtimeInputs = [ name = "text-generation-inference";
server runtimeInputs = [
router server
]; router
text = '' ];
${launcher}/bin/text-generation-launcher "$@" text = ''
''; ${launcher}/bin/text-generation-launcher "$@"
'';
};
dockerImage = pkgs.dockerTools.buildLayeredImage {
name = "tgi-docker";
tag = "latest";
config = {
EntryPoint = [ "${default}/bin/text-generation-inference" ];
Env = [
"HF_HOME=/data"
"PORT=80"
];
};
contents = [ pkgs.cacert ];
};
}; };
} }
); );