huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-09 19:34:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge 75ebb228f4 into 06d9d88b95

Browse Source
This commit is contained in:
Daniël de Kok 2025-08-29 11:04:53 +07:00 committed by GitHub
commit f858028741
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 358 additions and 232 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
flake.lock
View File

@ -586,15 +586,16 @@
"nixpkgs": "nixpkgs_6" "nixpkgs": "nixpkgs_6"
}, },
"locked": { "locked": {
"lastModified": 1747919133, "lastModified": 1751978820,
"narHash": "sha256-VvF1naQOvv7yulQ5/cDiaxkNxlh1Y84QMZnderv1szk=", "narHash": "sha256-MzZszOGoIsn/wleAuHbF7xd5tRuMxXQMbaDYijDfPEY=",
"owner": "huggingface", "owner": "huggingface",
"repo": "hf-nix", "repo": "hf-nix",
"rev": "9c71e026d6c7c8588ef85a5f7c77f57d598e038c", "rev": "8fd72d076f14ee0d6a3800d9e4434a70a7ea48f7",
"type": "github" "type": "github"
}, },
"original": { "original": {
"owner": "huggingface", "owner": "huggingface",
"ref": "quantization-0.1.0",
"repo": "hf-nix", "repo": "hf-nix",
"type": "github" "type": "github"
} }

4
flake.nix
View File

@ -5,7 +5,7 @@
inputs.nixpkgs.follows = "hf-nix/nixpkgs"; inputs.nixpkgs.follows = "hf-nix/nixpkgs";
}; };
nix-filter.url = "github:numtide/nix-filter"; nix-filter.url = "github:numtide/nix-filter";
hf-nix.url = "github:huggingface/hf-nix"; hf-nix.url = "github:huggingface/hf-nix/quantization-0.1.0";
nixpkgs.follows = "hf-nix/nixpkgs"; nixpkgs.follows = "hf-nix/nixpkgs";
flake-utils.url = "github:numtide/flake-utils"; flake-utils.url = "github:numtide/flake-utils";
rust-overlay = { rust-overlay = {
@ -33,7 +33,7 @@
}; };
pkgs = import nixpkgs { pkgs = import nixpkgs {
inherit system; inherit system;
inherit (hf-nix.lib) config; config = hf-nix.lib.config system;
overlays = [ overlays = [
rust-overlay.overlays.default rust-overlay.overlays.default
hf-nix.overlays.default hf-nix.overlays.default

308
integration-tests/models/__snapshots__/test_compressed_tensors_w8a8_int_dynamic_weight/test_compressed_tensors_w8a8_int_dynamic_weight.json
View File

@ -1,469 +1,613 @@
{ {
"details": { "details": {
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "eos_token", "finish_reason": "length",
"generated_tokens": 76, "generated_tokens": 100,
"prefill": [], "prefill": [],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
"id": 18183, "id": 18183,
"logprob": -1.5195312, "logprob": -1.5371094,
"special": false, "special": false,
"text": " Deep" "text": " Deep"
}, },
{ {
"id": 6832, "id": 6832,
"logprob": -0.06817627, "logprob": -0.08483887,
"special": false, "special": false,
"text": " learning" "text": " learning"
}, },
{ {
"id": 374, "id": 374,
"logprob": -0.13122559, "logprob": -0.13378906,
"special": false, "special": false,
"text": " is" "text": " is"
}, },
{ {
"id": 264, "id": 264,
"logprob": -0.13415527, "logprob": -0.14562988,
"special": false, "special": false,
"text": " a" "text": " a"
}, },
{ {
"id": 25993, "id": 25993,
"logprob": -0.8769531, "logprob": -0.78222656,
"special": false, "special": false,
"text": " subset" "text": " subset"
}, },
{ {
"id": 315, "id": 315,
"logprob": -0.0011396408, "logprob": -0.0013389587,
"special": false, "special": false,
"text": " of" "text": " of"
}, },
{ {
"id": 5662, "id": 5662,
"logprob": -0.16442871, "logprob": -0.15234375,
"special": false, "special": false,
"text": " machine" "text": " machine"
}, },
{ {
"id": 6832, "id": 6832,
"logprob": -0.0026416779, "logprob": -0.0018444061,
"special": false, "special": false,
"text": " learning" "text": " learning"
}, },
{ {
"id": 429, "id": 429,
"logprob": -0.48754883, "logprob": -0.45507812,
"special": false, "special": false,
"text": " that" "text": " that"
}, },
{ {
"id": 5711, "id": 5711,
"logprob": -1.2294922, "logprob": -1.1435547,
"special": false, "special": false,
"text": " uses" "text": " uses"
}, },
{ {
"id": 29728, "id": 29728,
"logprob": -0.66503906, "logprob": -0.78515625,
"special": false, "special": false,
"text": " neural" "text": " neural"
}, },
{ {
"id": 14155, "id": 14155,
"logprob": -0.02960205, "logprob": -0.022445679,
"special": false, "special": false,
"text": " networks" "text": " networks"
}, },
{ {
"id": 311, "id": 311,
"logprob": -0.7236328, "logprob": -0.6767578,
"special": false, "special": false,
"text": " to" "text": " to"
}, },
{ {
"id": 3960, "id": 3960,
"logprob": -1.1914062, "logprob": -1.1796875,
"special": false, "special": false,
"text": " learn" "text": " learn"
}, },
{ {
"id": 504, "id": 504,
"logprob": -0.7089844, "logprob": -0.77441406,
"special": false, "special": false,
"text": " from" "text": " from"
}, },
{ {
"id": 821, "id": 821,
"logprob": -0.7729492, "logprob": -0.67089844,
"special": false, "special": false,
"text": " data" "text": " data"
}, },
{ {
"id": 13, "id": 13,
"logprob": -0.7836914, "logprob": -0.64404297,
"special": false, "special": false,
"text": "." "text": "."
}, },
{ {
"id": 1084, "id": 1084,
"logprob": -0.9941406, "logprob": -1.1582031,
"special": false, "special": false,
"text": " It" "text": " It"
}, },
{ {
"id": 374, "id": 374,
"logprob": -0.52441406, "logprob": -0.5810547,
"special": false, "special": false,
"text": " is" "text": " is"
}, },
{ {
"id": 264, "id": 264,
"logprob": -0.9511719, "logprob": -1.1416016,
"special": false, "special": false,
"text": " a" "text": " a"
}, },
{ {
"id": 943, "id": 943,
"logprob": -0.8642578, "logprob": -0.9873047,
"special": false, "special": false,
"text": " type" "text": " type"
}, },
{ {
"id": 315, "id": 315,
"logprob": -0.00030231476, "logprob": -0.0001975298,
"special": false, "special": false,
"text": " of" "text": " of"
}, },
{ {
"id": 20443, "id": 20443,
"logprob": -0.14416504, "logprob": -0.22302246,
"special": false, "special": false,
"text": " artificial" "text": " artificial"
}, },
{ {
"id": 11229, "id": 11229,
"logprob": -0.013824463, "logprob": -0.012550354,
"special": false, "special": false,
"text": " intelligence" "text": " intelligence"
}, },
{ {
"id": 429, "id": 429,
"logprob": -0.18762207, "logprob": -0.2130127,
"special": false, "special": false,
"text": " that" "text": " that"
}, },
{ {
"id": 646, "id": 646,
"logprob": -1.0087891, "logprob": -1.1347656,
"special": false, "special": false,
"text": " can" "text": " can"
}, },
{ {
"id": 3960, "id": 3960,
"logprob": -0.90234375, "logprob": -0.97802734,
"special": false, "special": false,
"text": " learn" "text": " learn"
}, },
{ {
"id": 504, "id": 504,
"logprob": -0.54345703, "logprob": -0.4489746,
"special": false, "special": false,
"text": " from" "text": " from"
}, },
{ {
"id": 323, "id": 323,
"logprob": -1.0400391, "logprob": -0.9038086,
"special": false, "special": false,
"text": " and" "text": " and"
}, },
{ {
"id": 1281, "id": 1281,
"logprob": -0.072509766, "logprob": -0.10961914,
"special": false, "special": false,
"text": " make" "text": " make"
}, },
{ {
"id": 19898, "id": 19898,
"logprob": -0.16516113, "logprob": -0.3503418,
"special": false, "special": false,
"text": " predictions" "text": " predictions"
}, },
{ {
"id": 389, "id": 389,
"logprob": -0.4416504, "logprob": -0.62939453,
"special": false, "special": false,
"text": " on" "text": " on"
}, },
{ {
"id": 3460, "id": 3460,
"logprob": -0.5385742, "logprob": -0.9458008,
"special": false, "special": false,
"text": " large" "text": " large"
}, },
{ {
"id": 14713, "id": 14713,
"logprob": -0.4387207, "logprob": -0.33813477,
"special": false, "special": false,
"text": " amounts" "text": " amounts"
}, },
{ {
"id": 315, "id": 315,
"logprob": -0.00015091896, "logprob": -0.00013554096,
"special": false, "special": false,
"text": " of" "text": " of"
}, },
{ {
"id": 821, "id": 821,
"logprob": -0.061431885, "logprob": -0.06390381,
"special": false, "special": false,
"text": " data" "text": " data"
}, },
{ {
"id": 13, "id": 13,
"logprob": -0.71875, "logprob": -0.6826172,
"special": false, "special": false,
"text": "." "text": "."
}, },
{ {
"id": 18183, "id": 18183,
"logprob": -0.23632812, "logprob": -0.3503418,
"special": false, "special": false,
"text": " Deep" "text": " Deep"
}, },
{ {
"id": 6832, "id": 6832,
"logprob": -0.0017204285, "logprob": -0.0020923615,
"special": false, "special": false,
"text": " learning" "text": " learning"
}, },
{ {
"id": 374, "id": 374,
"logprob": -1.1738281, "logprob": -1.1357422,
"special": false, "special": false,
"text": " is" "text": " is"
}, },
{ {
"id": 1483, "id": 1483,
"logprob": -0.61083984, "logprob": -0.76416016,
"special": false, "special": false,
"text": " used" "text": " used"
}, },
{ {
"id": 304, "id": 304,
"logprob": -0.035003662, "logprob": -0.04458618,
"special": false, "special": false,
"text": " in" "text": " in"
}, },
{ {
"id": 264, "id": 264,
"logprob": -0.118652344, "logprob": -0.09295654,
"special": false, "special": false,
"text": " a" "text": " a"
}, },
{ {
"id": 8045, "id": 8045,
"logprob": -0.42016602, "logprob": -0.54003906,
"special": false, "special": false,
"text": " variety" "text": " variety"
}, },
{ {
"id": 315, "id": 315,
"logprob": -1.6212463e-05, "logprob": -1.6450882e-05,
"special": false, "special": false,
"text": " of" "text": " of"
}, },
{ {
"id": 8357, "id": 8357,
"logprob": -0.1315918, "logprob": -0.095947266,
"special": false, "special": false,
"text": " applications" "text": " applications"
}, },
{ {
"id": 11, "id": 11,
"logprob": -0.12915039, "logprob": -0.10650635,
"special": false, "special": false,
"text": "," "text": ","
}, },
{ {
"id": 2670, "id": 2670,
"logprob": -0.12463379, "logprob": -0.079589844,
"special": false, "special": false,
"text": " including" "text": " including"
}, },
{ {
"id": 2168, "id": 2168,
"logprob": -0.37402344, "logprob": -0.40551758,
"special": false, "special": false,
"text": " image" "text": " image"
}, },
{ {
"id": 323, "id": 323,
"logprob": -0.1451416, "logprob": -0.13012695,
"special": false, "special": false,
"text": " and" "text": " and"
}, },
{ {
"id": 8806, "id": 8806,
"logprob": -0.028869629, "logprob": -0.02720642,
"special": false, "special": false,
"text": " speech" "text": " speech"
}, },
{ {
"id": 17843, "id": 17843,
"logprob": -0.00024068356, "logprob": -0.00020062923,
"special": false, "special": false,
"text": " recognition" "text": " recognition"
}, },
{ {
"id": 11, "id": 11,
"logprob": -0.00031018257, "logprob": -0.00056505203,
"special": false, "special": false,
"text": "," "text": ","
}, },
{ {
"id": 5810, "id": 5810,
"logprob": -0.019821167, "logprob": -0.022247314,
"special": false, "special": false,
"text": " natural" "text": " natural"
}, },
{ {
"id": 4128, "id": 4128,
"logprob": -0.00012528896, "logprob": -0.00017559528,
"special": false, "special": false,
"text": " language" "text": " language"
}, },
{ {
"id": 8692, "id": 8692,
"logprob": -0.00089263916, "logprob": -0.0007171631,
"special": false, "special": false,
"text": " processing" "text": " processing"
}, },
{ {
"id": 11, "id": 11,
"logprob": -0.00073862076, "logprob": -0.0007882118,
"special": false, "special": false,
"text": "," "text": ","
}, },
{ {
"id": 323, "id": 323,
"logprob": -0.040161133, "logprob": -0.027862549,
"special": false, "special": false,
"text": " and" "text": " and"
}, },
{ {
"id": 38193, "id": 38193,
"logprob": -0.4519043, "logprob": -0.39111328,
"special": false, "special": false,
"text": " autonomous" "text": " autonomous"
}, },
{ {
"id": 11474, "id": 11474,
"logprob": -0.39941406, "logprob": -0.38427734,
"special": false, "special": false,
"text": " vehicles" "text": " vehicles"
}, },
{ {
"id": 13, "id": 13,
"logprob": -0.21166992, "logprob": -0.23461914,
"special": false, "special": false,
"text": "." "text": "."
}, },
{ {
"id": 1084, "id": 1084,
"logprob": -0.9082031, "logprob": -1.0439453,
"special": false, "special": false,
"text": " It" "text": " It"
}, },
{ {
"id": 374, "id": 374,
"logprob": -0.44213867, "logprob": -0.44580078,
"special": false, "special": false,
"text": " is" "text": " is"
}, },
{ {
"id": 264, "id": 264,
"logprob": -1.2177734, "logprob": -0.86865234,
"special": false, "special": false,
"text": " a" "text": " a"
}, },
{ {
"id": 18512, "id": 18512,
"logprob": -0.5205078, "logprob": -0.5263672,
"special": false, "special": false,
"text": " rapidly" "text": " rapidly"
}, },
{ {
"id": 7826, "id": 7826,
"logprob": -0.15332031, "logprob": -0.15881348,
"special": false, "special": false,
"text": " growing" "text": " growing"
}, },
{ {
"id": 2070, "id": 2070,
"logprob": -0.0039978027, "logprob": -0.0044059753,
"special": false, "special": false,
"text": " field" "text": " field"
}, },
{ {
"id": 448, "id": 448,
"logprob": -0.9091797, "logprob": -0.921875,
"special": false, "special": false,
"text": " with" "text": " with"
}, },
{ {
"id": 1657, "id": 1657,
"logprob": -0.17114258, "logprob": -0.18737793,
"special": false, "special": false,
"text": " many" "text": " many"
}, },
{ {
"id": 4650, "id": 4650,
"logprob": -0.70703125, "logprob": -0.8857422,
"special": false, "special": false,
"text": " potential" "text": " potential"
}, },
{ {
"id": 8357, "id": 8357,
"logprob": -0.025131226, "logprob": -0.036193848,
"special": false, "special": false,
"text": " applications" "text": " applications"
}, },
{ {
"id": 304, "id": 304,
"logprob": -0.6699219, "logprob": -0.65283203,
"special": false, "special": false,
"text": " in" "text": " in"
}, },
{ {
"id": 279, "id": 279,
"logprob": -0.35205078, "logprob": -0.4411621,
"special": false, "special": false,
"text": " the" "text": " the"
}, },
{ {
"id": 3853, "id": 3853,
"logprob": -0.049194336, "logprob": -0.059326172,
"special": false, "special": false,
"text": " future" "text": " future"
}, },
{ {
"id": 13, "id": 13,
"logprob": -0.21972656, "logprob": -0.23278809,
"special": false, "special": false,
"text": "." "text": "."
}, },
{ {
"id": 151643, "id": 3555,
"logprob": -2.0019531, "logprob": -1.90625,
"special": true, "special": false,
"text": "<|endoftext|>" "text": " What"
},
{
"id": 525,
"logprob": -0.48291016,
"special": false,
"text": " are"
},
{
"id": 1045,
"logprob": -0.1484375,
"special": false,
"text": " some"
},
{
"id": 10295,
"logprob": -1.4072266,
"special": false,
"text": " examples"
},
{
"id": 315,
"logprob": -0.00091028214,
"special": false,
"text": " of"
},
{
"id": 5538,
"logprob": -0.47192383,
"special": false,
"text": " deep"
},
{
"id": 6832,
"logprob": -0.0005393028,
"special": false,
"text": " learning"
},
{
"id": 8357,
"logprob": -0.33569336,
"special": false,
"text": " applications"
},
{
"id": 30,
"logprob": -0.19299316,
"special": false,
"text": "?"
},
{
"id": 2619,
"logprob": -1.3320312,
"special": false,
"text": " There"
},
{
"id": 525,
"logprob": -0.0027637482,
"special": false,
"text": " are"
},
{
"id": 1657,
"logprob": -0.0574646,
"special": false,
"text": " many"
},
{
"id": 10295,
"logprob": -0.093811035,
"special": false,
"text": " examples"
},
{
"id": 315,
"logprob": -0.000106692314,
"special": false,
"text": " of"
},
{
"id": 5538,
"logprob": -0.013023376,
"special": false,
"text": " deep"
},
{
"id": 6832,
"logprob": -7.081032e-05,
"special": false,
"text": " learning"
},
{
"id": 8357,
"logprob": -0.010604858,
"special": false,
"text": " applications"
},
{
"id": 11,
"logprob": -0.28125,
"special": false,
"text": ","
},
{
"id": 2670,
"logprob": -0.5209961,
"special": false,
"text": " including"
},
{
"id": 1447,
"logprob": -0.8300781,
"special": false,
"text": ":\n\n"
},
{
"id": 220,
"logprob": -0.8071289,
"special": false,
"text": " "
},
{
"id": 353,
"logprob": -0.07385254,
"special": false,
"text": " *"
},
{
"id": 4654,
"logprob": -0.12548828,
"special": false,
"text": " Image"
},
{
"id": 17843,
"logprob": -0.4790039,
"special": false,
"text": " recognition"
},
{
"id": 25,
"logprob": -0.25634766,
"special": false,
"text": ":"
} }
], ],
"top_tokens": null "top_tokens": null
}, },
"generated_text": " Deep learning is a subset of machine learning that uses neural networks to learn from data. It is a type of artificial intelligence that can learn from and make predictions on large amounts of data. Deep learning is used in a variety of applications, including image and speech recognition, natural language processing, and autonomous vehicles. It is a rapidly growing field with many potential applications in the future." "generated_text": " Deep learning is a subset of machine learning that uses neural networks to learn from data. It is a type of artificial intelligence that can learn from and make predictions on large amounts of data. Deep learning is used in a variety of applications, including image and speech recognition, natural language processing, and autonomous vehicles. It is a rapidly growing field with many potential applications in the future. What are some examples of deep learning applications? There are many examples of deep learning applications, including:\n\n * Image recognition:"
} }

26
integration-tests/models/__snapshots__/test_compressed_tensors_w8a8_int_dynamic_weight/test_compressed_tensors_w8a8_int_dynamic_weight_all_params.json
View File

@ -8,25 +8,25 @@
"tokens": [ "tokens": [
{ {
"id": 5267, "id": 5267,
"logprob": -1.1464844, "logprob": -1.0410156,
"special": false, "special": false,
"text": "?\n" "text": "?\n"
}, },
{ {
"id": 33464, "id": 33464,
"logprob": -0.83203125, "logprob": -0.6147461,
"special": false, "special": false,
"text": "Deep" "text": "Deep"
}, },
{ {
"id": 20909, "id": 20909,
"logprob": -0.5625, "logprob": -0.5229492,
"special": false, "special": false,
"text": " Learning" "text": " Learning"
}, },
{ {
"id": 320, "id": 320,
"logprob": -2.1464844, "logprob": -1.7451172,
"special": false, "special": false,
"text": " (" "text": " ("
}, },
@ -38,36 +38,36 @@
}, },
{ {
"id": 701, "id": 701,
"logprob": -2.2089844, "logprob": -2.2382812,
"special": false, "special": false,
"text": ")," "text": "),"
}, },
{ {
"id": 476, "id": 476,
"logprob": -0.27368164, "logprob": -0.22546387,
"special": false, "special": false,
"text": " or" "text": " or"
}, },
{ {
"id": 20443, "id": 20443,
"logprob": -0.09442139, "logprob": -0.16967773,
"special": false, "special": false,
"text": " artificial" "text": " artificial"
}, },
{ {
"id": 29728, "id": 11229,
"logprob": 0.0, "logprob": -2.265625,
"special": false, "special": false,
"text": " neural" "text": " intelligence"
}, },
{ {
"id": 14155, "id": 11,
"logprob": 0.0, "logprob": 0.0,
"special": false, "special": false,
"text": " networks" "text": ","
} }
], ],
"top_tokens": null "top_tokens": null
}, },
"generated_text": "What is deep learning?\nDeep Learning (DL), or artificial neural networks" "generated_text": "What is deep learning?\nDeep Learning (DL), or artificial intelligence,"
} }

80
integration-tests/models/__snapshots__/test_compressed_tensors_w8a8_int_dynamic_weight/test_compressed_tensors_w8a8_int_dynamic_weight_load.json
View File

@ -9,61 +9,61 @@
"tokens": [ "tokens": [
{ {
"id": 18183, "id": 18183,
"logprob": -1.5195312, "logprob": -1.5371094,
"special": false, "special": false,
"text": " Deep" "text": " Deep"
}, },
{ {
"id": 6832, "id": 6832,
"logprob": -0.06817627, "logprob": -0.08483887,
"special": false, "special": false,
"text": " learning" "text": " learning"
}, },
{ {
"id": 374, "id": 374,
"logprob": -0.13122559, "logprob": -0.13378906,
"special": false, "special": false,
"text": " is" "text": " is"
}, },
{ {
"id": 264, "id": 264,
"logprob": -0.13415527, "logprob": -0.14562988,
"special": false, "special": false,
"text": " a" "text": " a"
}, },
{ {
"id": 25993, "id": 25993,
"logprob": -0.87353516, "logprob": -0.78222656,
"special": false, "special": false,
"text": " subset" "text": " subset"
}, },
{ {
"id": 315, "id": 315,
"logprob": -0.0011396408, "logprob": -0.0013389587,
"special": false, "special": false,
"text": " of" "text": " of"
}, },
{ {
"id": 5662, "id": 5662,
"logprob": -0.16442871, "logprob": -0.15234375,
"special": false, "special": false,
"text": " machine" "text": " machine"
}, },
{ {
"id": 6832, "id": 6832,
"logprob": -0.0026416779, "logprob": -0.0018444061,
"special": false, "special": false,
"text": " learning" "text": " learning"
}, },
{ {
"id": 429, "id": 429,
"logprob": -0.48754883, "logprob": -0.45507812,
"special": false, "special": false,
"text": " that" "text": " that"
}, },
{ {
"id": 5711, "id": 5711,
"logprob": -1.2294922, "logprob": -1.1435547,
"special": false, "special": false,
"text": " uses" "text": " uses"
} }
@ -82,61 +82,61 @@
"tokens": [ "tokens": [
{ {
"id": 18183, "id": 18183,
"logprob": -1.5195312, "logprob": -1.5371094,
"special": false, "special": false,
"text": " Deep" "text": " Deep"
}, },
{ {
"id": 6832, "id": 6832,
"logprob": -0.06817627, "logprob": -0.08483887,
"special": false, "special": false,
"text": " learning" "text": " learning"
}, },
{ {
"id": 374, "id": 374,
"logprob": -0.13122559, "logprob": -0.13378906,
"special": false, "special": false,
"text": " is" "text": " is"
}, },
{ {
"id": 264, "id": 264,
"logprob": -0.13415527, "logprob": -0.14562988,
"special": false, "special": false,
"text": " a" "text": " a"
}, },
{ {
"id": 25993, "id": 25993,
"logprob": -0.87353516, "logprob": -0.78222656,
"special": false, "special": false,
"text": " subset" "text": " subset"
}, },
{ {
"id": 315, "id": 315,
"logprob": -0.0011396408, "logprob": -0.0013389587,
"special": false, "special": false,
"text": " of" "text": " of"
}, },
{ {
"id": 5662, "id": 5662,
"logprob": -0.16442871, "logprob": -0.15234375,
"special": false, "special": false,
"text": " machine" "text": " machine"
}, },
{ {
"id": 6832, "id": 6832,
"logprob": -0.0026416779, "logprob": -0.0018444061,
"special": false, "special": false,
"text": " learning" "text": " learning"
}, },
{ {
"id": 429, "id": 429,
"logprob": -0.48754883, "logprob": -0.45507812,
"special": false, "special": false,
"text": " that" "text": " that"
}, },
{ {
"id": 5711, "id": 5711,
"logprob": -1.2294922, "logprob": -1.1435547,
"special": false, "special": false,
"text": " uses" "text": " uses"
} }
@ -155,61 +155,61 @@
"tokens": [ "tokens": [
{ {
"id": 18183, "id": 18183,
"logprob": -1.5195312, "logprob": -1.5371094,
"special": false, "special": false,
"text": " Deep" "text": " Deep"
}, },
{ {
"id": 6832, "id": 6832,
"logprob": -0.06817627, "logprob": -0.08483887,
"special": false, "special": false,
"text": " learning" "text": " learning"
}, },
{ {
"id": 374, "id": 374,
"logprob": -0.13122559, "logprob": -0.13378906,
"special": false, "special": false,
"text": " is" "text": " is"
}, },
{ {
"id": 264, "id": 264,
"logprob": -0.13415527, "logprob": -0.14562988,
"special": false, "special": false,
"text": " a" "text": " a"
}, },
{ {
"id": 25993, "id": 25993,
"logprob": -0.87353516, "logprob": -0.78222656,
"special": false, "special": false,
"text": " subset" "text": " subset"
}, },
{ {
"id": 315, "id": 315,
"logprob": -0.0011396408, "logprob": -0.0013389587,
"special": false, "special": false,
"text": " of" "text": " of"
}, },
{ {
"id": 5662, "id": 5662,
"logprob": -0.16442871, "logprob": -0.15234375,
"special": false, "special": false,
"text": " machine" "text": " machine"
}, },
{ {
"id": 6832, "id": 6832,
"logprob": -0.0026416779, "logprob": -0.0018444061,
"special": false, "special": false,
"text": " learning" "text": " learning"
}, },
{ {
"id": 429, "id": 429,
"logprob": -0.48754883, "logprob": -0.45507812,
"special": false, "special": false,
"text": " that" "text": " that"
}, },
{ {
"id": 5711, "id": 5711,
"logprob": -1.2294922, "logprob": -1.1435547,
"special": false, "special": false,
"text": " uses" "text": " uses"
} }
@ -228,61 +228,61 @@
"tokens": [ "tokens": [
{ {
"id": 18183, "id": 18183,
"logprob": -1.5195312, "logprob": -1.5371094,
"special": false, "special": false,
"text": " Deep" "text": " Deep"
}, },
{ {
"id": 6832, "id": 6832,
"logprob": -0.06817627, "logprob": -0.08483887,
"special": false, "special": false,
"text": " learning" "text": " learning"
}, },
{ {
"id": 374, "id": 374,
"logprob": -0.13122559, "logprob": -0.13378906,
"special": false, "special": false,
"text": " is" "text": " is"
}, },
{ {
"id": 264, "id": 264,
"logprob": -0.13415527, "logprob": -0.14562988,
"special": false, "special": false,
"text": " a" "text": " a"
}, },
{ {
"id": 25993, "id": 25993,
"logprob": -0.87353516, "logprob": -0.78222656,
"special": false, "special": false,
"text": " subset" "text": " subset"
}, },
{ {
"id": 315, "id": 315,
"logprob": -0.0011396408, "logprob": -0.0013389587,
"special": false, "special": false,
"text": " of" "text": " of"
}, },
{ {
"id": 5662, "id": 5662,
"logprob": -0.16442871, "logprob": -0.15234375,
"special": false, "special": false,
"text": " machine" "text": " machine"
}, },
{ {
"id": 6832, "id": 6832,
"logprob": -0.0026416779, "logprob": -0.0018444061,
"special": false, "special": false,
"text": " learning" "text": " learning"
}, },
{ {
"id": 429, "id": 429,
"logprob": -0.48754883, "logprob": -0.45507812,
"special": false, "special": false,
"text": " that" "text": " that"
}, },
{ {
"id": 5711, "id": 5711,
"logprob": -1.2294922, "logprob": -1.1435547,
"special": false, "special": false,
"text": " uses" "text": " uses"
} }

12
integration-tests/models/__snapshots__/test_flash_llama_fp8_kv_cache/test_flash_llama_fp8_kv_cache_all_params.json
View File

@ -14,25 +14,25 @@
}, },
{ {
"id": 34564, "id": 34564,
"logprob": -0.12512207, "logprob": -0.12695312,
"special": false, "special": false,
"text": "Deep" "text": "Deep"
}, },
{ {
"id": 6975, "id": 6975,
"logprob": 0.0, "logprob": -0.10632324,
"special": false, "special": false,
"text": " learning" "text": " learning"
}, },
{ {
"id": 320, "id": 320,
"logprob": -0.23840332, "logprob": -0.22546387,
"special": false, "special": false,
"text": " (" "text": " ("
}, },
{ {
"id": 16931, "id": 16931,
"logprob": -2.0175781, "logprob": -1.8837891,
"special": false, "special": false,
"text": "DL" "text": "DL"
}, },
@ -44,7 +44,7 @@
}, },
{ {
"id": 374, "id": 374,
"logprob": -0.8613281, "logprob": -0.8798828,
"special": false, "special": false,
"text": " is" "text": " is"
}, },
@ -56,7 +56,7 @@
}, },
{ {
"id": 1207, "id": 1207,
"logprob": -1.2451172, "logprob": -1.2675781,
"special": false, "special": false,
"text": " sub" "text": " sub"
}, },

8
integration-tests/models/test_compressed_tensors_w8a8_int_dynamic_weight.py
View File

@ -28,15 +28,15 @@ async def test_compressed_tensors_w8a8_int_dynamic_weight(
response = await compressed_tensors_w8a8_int_dynamic_weight.generate( response = await compressed_tensors_w8a8_int_dynamic_weight.generate(
"What is deep learning?", "What is deep learning?",
# prefer a longer response than the default, allow the llm to end generation # prefer a longer response than the default, allow the llm to end generation
max_new_tokens=1000, max_new_tokens=100,
decoder_input_details=True, decoder_input_details=True,
) )
assert ( assert (
response.generated_text response.generated_text
== " Deep learning is a subset of machine learning that uses neural networks to learn from data. It is a type of artificial intelligence that can learn from and make predictions on large amounts of data. Deep learning is used in a variety of applications, including image and speech recognition, natural language processing, and autonomous vehicles. It is a rapidly growing field with many potential applications in the future." == " Deep learning is a subset of machine learning that uses neural networks to learn from data. It is a type of artificial intelligence that can learn from and make predictions on large amounts of data. Deep learning is used in a variety of applications, including image and speech recognition, natural language processing, and autonomous vehicles. It is a rapidly growing field with many potential applications in the future. What are some examples of deep learning applications? There are many examples of deep learning applications, including:\n\n * Image recognition:"
) )
assert response.details.generated_tokens == 76 assert response.details.generated_tokens == 100
assert response == response_snapshot assert response == response_snapshot
@ -65,7 +65,7 @@ async def test_compressed_tensors_w8a8_int_dynamic_weight_all_params(
assert response.details.generated_tokens == 10 assert response.details.generated_tokens == 10
assert ( assert (
response.generated_text response.generated_text
== "What is deep learning?\nDeep Learning (DL), or artificial neural networks" == "What is deep learning?\nDeep Learning (DL), or artificial intelligence,"
) )
assert response == response_snapshot assert response == response_snapshot

52
server/kernels.lock
View File

@ -223,82 +223,58 @@
}, },
{ {
"repo_id": "kernels-community/quantization", "repo_id": "kernels-community/quantization",
"sha": "6470f9b005797e00279eb9103463dfe0f8b7da00", "sha": "b65f8ab1edf7f2f3174f9583de38e3c594c80d20",
"variants": { "variants": {
"torch25-cxx11-cu118-x86_64-linux": {
"hash": "sha256-f52c9b1a7cd98fb389c6d2a0b22a293cb36eb96af3a624f5aec761735861c96d",
"hash_type": "git_lfs_concat"
},
"torch25-cxx11-cu121-x86_64-linux": {
"hash": "sha256-e5f0da343363a562ce52f147a9534cd54a3efa90e70671f606cc2516f02a3876",
"hash_type": "git_lfs_concat"
},
"torch25-cxx11-cu124-x86_64-linux": {
"hash": "sha256-caad9300c155faf79c26426f10951ba75f931a05e741a5b39a24b064daabc040",
"hash_type": "git_lfs_concat"
},
"torch25-cxx98-cu118-x86_64-linux": {
"hash": "sha256-4fc87893de14a29ba4b55f5026ea05ec5901c0b52abd5ebae681ea0b791e858c",
"hash_type": "git_lfs_concat"
},
"torch25-cxx98-cu121-x86_64-linux": {
"hash": "sha256-72c975ea63fc524a38fcee5b2dbdb566eff0a0ea546ee5756441d04908e4e896",
"hash_type": "git_lfs_concat"
},
"torch25-cxx98-cu124-x86_64-linux": {
"hash": "sha256-28c5510e3b07eae2b3846b880f6111da65df024e1f24f81077d187a97c015364",
"hash_type": "git_lfs_concat"
},
"torch26-cxx11-cu118-x86_64-linux": { "torch26-cxx11-cu118-x86_64-linux": {
"hash": "sha256-8444cf77686578a6b0f7e2fd29bf2783ba120ebf7df41573f61d2521fd0acc10", "hash": "sha256-86c9ea9d12090a1bd457350acf85cbabac7af7253b6ce2603f8a7c6a3f03058b",
"hash_type": "git_lfs_concat" "hash_type": "git_lfs_concat"
}, },
"torch26-cxx11-cu124-x86_64-linux": { "torch26-cxx11-cu124-x86_64-linux": {
"hash": "sha256-6ea8e00625b5fe799fbe407e7de0fc08228cac26f9bbed2d70a6500026fe3bab", "hash": "sha256-03a2d88dd1d725e6c7cb7781f53d92006570fd12d226e1d9f3b2c1aed980eed2",
"hash_type": "git_lfs_concat" "hash_type": "git_lfs_concat"
}, },
"torch26-cxx11-cu126-aarch64-linux": { "torch26-cxx11-cu126-aarch64-linux": {
"hash": "sha256-0b8b8afbdaf9aa533895cb9e884e3ad3e9a34d483f05a1bbde1b8902f9dbeb0f", "hash": "sha256-c2907b0538618ec896d495b9a6bb62c9076d5c0aa7233d5524aa408379042b29",
"hash_type": "git_lfs_concat" "hash_type": "git_lfs_concat"
}, },
"torch26-cxx11-cu126-x86_64-linux": { "torch26-cxx11-cu126-x86_64-linux": {
"hash": "sha256-e115e855d7ca4b97787f04c88e128432256c6b43d4823fb8889ab9985dc4cf36", "hash": "sha256-e5a1790e97648bf90dd269606298bbf1ee17e8504a6c4d2a6416190266c9f57a",
"hash_type": "git_lfs_concat" "hash_type": "git_lfs_concat"
}, },
"torch26-cxx98-cu118-x86_64-linux": { "torch26-cxx98-cu118-x86_64-linux": {
"hash": "sha256-509f08c48a05584cc85c058607277fcbe3193e6cc61846dd2416d39e27c1d68e", "hash": "sha256-6934e8f14d95fad43603b08be07e37102bf70890d634c062fd6064cc7b83d718",
"hash_type": "git_lfs_concat" "hash_type": "git_lfs_concat"
}, },
"torch26-cxx98-cu124-x86_64-linux": { "torch26-cxx98-cu124-x86_64-linux": {
"hash": "sha256-a10236bffd435296c736ae2762ab0836da2421297e46b377368a17b39d70c27b", "hash": "sha256-9adda61755e617db96fa30542dd305dc612ccac0ddcf9fb4906f74ed51770354",
"hash_type": "git_lfs_concat" "hash_type": "git_lfs_concat"
}, },
"torch26-cxx98-cu126-aarch64-linux": { "torch26-cxx98-cu126-aarch64-linux": {
"hash": "sha256-ca2cb56f3eea4c399a61e21ba9b577d718b250aa60a13f42f01019ddd5cd8b0c", "hash": "sha256-9c389fd1c556edd2db7080bff0f78b03a63ac8382c73c79671c557b28a1c7e72",
"hash_type": "git_lfs_concat" "hash_type": "git_lfs_concat"
}, },
"torch26-cxx98-cu126-x86_64-linux": { "torch26-cxx98-cu126-x86_64-linux": {
"hash": "sha256-8fcd62d8243a30b63a03751cc0c15d24f6e00e43eae79f7281627f24e078bf9a", "hash": "sha256-4a86f8a5609fd35b81bebee4eb25c2d0733537e2a09011b8e704271470f3b0a3",
"hash_type": "git_lfs_concat" "hash_type": "git_lfs_concat"
}, },
"torch27-cxx11-cu118-x86_64-linux": { "torch27-cxx11-cu118-x86_64-linux": {
"hash": "sha256-60f5807ee3da937c57c1b6080c30632305aa4875ed5a52bf4e81968770b61b13", "hash": "sha256-39f6740ee44bf8fae873bbe8aed6f3596e59ca069e73c0b77d5e344ff2d5f7b7",
"hash_type": "git_lfs_concat" "hash_type": "git_lfs_concat"
}, },
"torch27-cxx11-cu126-aarch64-linux": { "torch27-cxx11-cu126-aarch64-linux": {
"hash": "sha256-64298b1713dc1d950915dc6569a06e2f541de3ed80aa5b32084246c1fdc7a958", "hash": "sha256-34dadff3669c3b256bf2637b76bac68991e3b6013f8a4ecaa51454cf2c80fa86",
"hash_type": "git_lfs_concat" "hash_type": "git_lfs_concat"
}, },
"torch27-cxx11-cu126-x86_64-linux": { "torch27-cxx11-cu126-x86_64-linux": {
"hash": "sha256-d9e219890dc28e8582ef21d6f81f2ebc361de218a86b742be63bc4714f102e5e", "hash": "sha256-f568e601daede1173e9952d32a835658f41791affe64069d7fd2d722002922e2",
"hash_type": "git_lfs_concat" "hash_type": "git_lfs_concat"
}, },
"torch27-cxx11-cu128-aarch64-linux": { "torch27-cxx11-cu128-aarch64-linux": {
"hash": "sha256-d72549f51aefcf020bc74262bbbccb78094638c5ab9adc8667873d247c1cce86", "hash": "sha256-f58b0b96c23b5340da08cd0db0221875c15b757640a783c0ad836bfc176799fa",
"hash_type": "git_lfs_concat" "hash_type": "git_lfs_concat"
}, },
"torch27-cxx11-cu128-x86_64-linux": { "torch27-cxx11-cu128-x86_64-linux": {
"hash": "sha256-d31ac5f87d7c7f62c63c72946479193aed467c9417c0acead5137e0e1fa968f8", "hash": "sha256-b2c9f8e42541fd24d553bb52172ac3bea8407c0bb32a3448632e4763a68f21f3",
"hash_type": "git_lfs_concat" "hash_type": "git_lfs_concat"
} }
} }

2
server/pyproject.toml
View File

@ -59,7 +59,7 @@ build-backend = "setuptools.build_meta"
"kernels-community/paged-attention" = ">=0.0.2" "kernels-community/paged-attention" = ">=0.0.2"
"kernels-community/moe" = ">=0.1.1" "kernels-community/moe" = ">=0.1.1"
"kernels-community/punica-sgmv" = ">=0.0.1" "kernels-community/punica-sgmv" = ">=0.0.1"
"kernels-community/quantization" = ">=0.0.3" "kernels-community/quantization" = ">=0.1.2"
"kernels-community/quantization-eetq" = ">=0.0.1" "kernels-community/quantization-eetq" = ">=0.0.1"
"kernels-community/rotary" = ">=0.0.1" "kernels-community/rotary" = ">=0.0.1"

11
server/text_generation_server/layers/compressed_tensors/w8a8_int.py
View File

@ -144,17 +144,6 @@ class W8A8IntLoader(WeightsLoader):
OtherT = TypeVar("OtherT") OtherT = TypeVar("OtherT")
def _get_tensor_or_else(
weights: Weights, prefix: str, other: OtherT
) -> Union[torch.Tensor, OtherT]:
# Even if a checkpoint uses e.g. zero-points, they can be elided:
# https://github.com/neuralmagic/compressed-tensors/blob/db6ccb25b265e8370813ecab5e95714a6728b5a6/src/compressed_tensors/compressors/quantized_compressors/base.py#L105
if weights.has_tensor(prefix):
return weights.get_tensor(prefix, to_dtype=False)
else:
return other
@dataclass @dataclass
class Int8Weight(Weight): class Int8Weight(Weight):
input_symmetric: bool input_symmetric: bool

25
server/text_generation_server/layers/marlin/fp8.py
View File

@ -76,15 +76,21 @@ class GPTQMarlinFP8Linear(nn.Module):
assert quantization is not None assert quantization is not None
A_flat = A.view(-1, A.shape[-1]) A_flat = A.view(-1, A.shape[-1])
C = quantization.fp8_marlin_gemm( C = quantization.gptq_marlin_gemm(
A_flat, a=A_flat,
self.qweight, c=None,
self.scales, b_q_weight=self.qweight,
self.workspace, b_scales=self.scales,
8, global_scale=None,
A_flat.shape[0], b_zeros=None,
self.scales.shape[1], g_idx=None,
A_flat.shape[1], perm=None,
workspace=self.workspace,
b_q_type=quantization.scalar_type.scalar_types.float8_e4m3fn,
size_m=A_flat.shape[0],
size_n=self.scales.shape[1],
size_k=A_flat.shape[1],
use_fp32_reduce=True,
) )
C = C.reshape(A.shape[:-1] + (self.scales.shape[1],)) C = C.reshape(A.shape[:-1] + (self.scales.shape[1],))
@ -143,5 +149,6 @@ def repack_fp8_for_marlin(weight: torch.Tensor, scales: torch.Tensor):
) )
scales = permute_scales(scales) scales = permute_scales(scales)
scales = quantization.marlin_utils_fp8.fp8_fused_exponent_bias_into_scales(scales)
return repacked, scales return repacked, scales

37
server/text_generation_server/layers/marlin/gptq.py
View File

@ -256,7 +256,7 @@ class GPTQMarlinWeight(Weight):
""" """
qweight: torch.Tensor qweight: torch.Tensor
qzeros: torch.Tensor qzeros: Optional[torch.Tensor]
scales: torch.Tensor scales: torch.Tensor
g_idx: torch.Tensor g_idx: torch.Tensor
perm: torch.Tensor perm: torch.Tensor
@ -268,6 +268,7 @@ class GPTQMarlinWeight(Weight):
assert self.scales.dtype in (torch.float16, torch.bfloat16) assert self.scales.dtype in (torch.float16, torch.bfloat16)
assert self.g_idx.dtype == torch.int32 assert self.g_idx.dtype == torch.int32
assert self.perm.dtype == torch.int32 assert self.perm.dtype == torch.int32
assert self.qzeros is None or self.qzeros.numel() > 0
def get_linear(self, bias: torch.Tensor): def get_linear(self, bias: torch.Tensor):
return GPTQMarlinLinear( return GPTQMarlinLinear(
@ -350,9 +351,6 @@ def repack_gptq_for_marlin(
qweight, perm, in_features, out_features, bits qweight, perm, in_features, out_features, bits
) )
if qzeros is None:
qzeros = torch.empty(0, dtype=torch.int, device=qweight.device)
scales = permute_scales(scales) scales = permute_scales(scales)
is_full_k = not (desc_act and groupsize != -1 and sharded_infeatures) is_full_k = not (desc_act and groupsize != -1 and sharded_infeatures)
@ -392,7 +390,7 @@ class GPTQMarlinLinear(nn.Module):
if weight.bits not in (4, 8): if weight.bits not in (4, 8):
raise ValueError("GPTQMarlinLinear only supports 4 and 8-bit quantization") raise ValueError("GPTQMarlinLinear only supports 4 and 8-bit quantization")
if weight.qzeros.numel() > 0: if weight.qzeros is not None:
if weight.bits == 4: if weight.bits == 4:
self.quant_type = quantization.scalar_types.uint4 self.quant_type = quantization.scalar_types.uint4
else: else:
@ -424,20 +422,21 @@ class GPTQMarlinLinear(nn.Module):
A_flat = A.view(-1, A.shape[-1]) A_flat = A.view(-1, A.shape[-1])
C = quantization.gptq_marlin_gemm( C = quantization.gptq_marlin_gemm(
A_flat, a=A_flat,
self.qweight, c=None,
self.scales, b_q_weight=self.qweight,
self.qzeros, b_scales=self.scales,
self.g_idx, global_scale=None,
self.perm, b_zeros=self.qzeros,
self.workspace, g_idx=self.g_idx,
self.quant_type, perm=self.perm,
A_flat.shape[0], workspace=self.workspace,
self.scales.shape[1], b_q_type=self.quant_type,
A_flat.shape[1], size_m=A_flat.shape[0],
self.is_full_k, size_n=self.scales.shape[1],
self.qzeros.numel() > 0, size_k=A_flat.shape[1],
True, is_k_full=self.is_full_k,
use_fp32_reduce=True,
) )
C = C.reshape(A.shape[:-1] + (self.scales.shape[1],)) C = C.reshape(A.shape[:-1] + (self.scales.shape[1],))

18
server/text_generation_server/layers/moe/gptq_marlin.py
View File

@ -202,9 +202,13 @@ def _pack_weight(
device=weight.qweight.device, device=weight.qweight.device,
) )
qzeros = torch.empty( qzeros = torch.empty(
(n_experts,) + weight.qzeros.shape, (n_experts,) + ((0,) if weight.qzeros is None else weight.qzeros.shape),
dtype=weight.qzeros.dtype, dtype=(
device=weight.qzeros.device, weight.qweight.dtype if weight.qzeros is None else weight.qzeros.dtype
),
device=(
weight.qweight.device if weight.qzeros is None else weight.qzeros.device
),
) )
scales = torch.empty( scales = torch.empty(
(n_experts,) + weight.scales.shape, (n_experts,) + weight.scales.shape,
@ -232,7 +236,13 @@ def _pack_weight(
) )
moe_weight.qweight[expert] = weight.qweight moe_weight.qweight[expert] = weight.qweight
moe_weight.qzeros[expert] = weight.qzeros moe_weight.qzeros[expert] = (
torch.zeros(
(0,), device=moe_weight.qzeros.device, dtype=moe_weight.qzeros.dtype
)
if weight.qzeros is None
else weight.qzeros
)
moe_weight.scales[expert] = weight.scales moe_weight.scales[expert] = weight.scales
moe_weight.g_idx[expert] = weight.g_idx moe_weight.g_idx[expert] = weight.g_idx
moe_weight.perm[expert] = weight.perm moe_weight.perm[expert] = weight.perm