mirror of
https://github.com/huggingface/text-generation-inference.git
synced 2025-04-21 23:12:07 +00:00
653193a942
* Improve support for GPUs with capability < 8 - For models that cannot use flashinfer, use flash-attn v1 + paged attention for models with a compute capability older than 8. - Disable prefix caching when using paged attention. - When using flash-attn v1, pass the key/value, rather than the cache, since v1 cannot use block tables. * nix: add flash-attn-v1 to the server environment * Move disabling prefix caching into the block of exceptions * Capability as `usize`s
55 lines
874 B
Nix
55 lines
874 B
Nix
{
|
|
mkShell,
|
|
openssl,
|
|
pkg-config,
|
|
protobuf,
|
|
python3,
|
|
pyright,
|
|
redocly,
|
|
ruff,
|
|
rust-bin,
|
|
server,
|
|
}:
|
|
|
|
mkShell {
|
|
buildInputs =
|
|
[
|
|
openssl.dev
|
|
pkg-config
|
|
(rust-bin.stable.latest.default.override {
|
|
extensions = [
|
|
"rust-analyzer"
|
|
"rust-src"
|
|
];
|
|
})
|
|
protobuf
|
|
pyright
|
|
redocly
|
|
ruff
|
|
]
|
|
++ (with python3.pkgs; [
|
|
venvShellHook
|
|
docker
|
|
pip
|
|
ipdb
|
|
click
|
|
pytest
|
|
pytest-asyncio
|
|
syrupy
|
|
]);
|
|
|
|
inputsFrom = [ server ];
|
|
|
|
venvDir = "./.venv";
|
|
|
|
postVenvCreation = ''
|
|
unset SOURCE_DATE_EPOCH
|
|
( cd server ; python -m pip install --no-dependencies -e . )
|
|
( cd clients/python ; python -m pip install --no-dependencies -e . )
|
|
'';
|
|
postShellHook = ''
|
|
unset SOURCE_DATE_EPOCH
|
|
export PATH=$PATH:~/.cargo/bin
|
|
'';
|
|
}
|