mirror of
https://github.com/huggingface/text-generation-inference.git
synced 2025-09-12 04:44:52 +00:00
Upgrading some stuff.
This commit is contained in:
Nicolas Patry
parent
af7d9f7b7d
commit
e128bc540b
58
flake.lock
58
flake.lock
@ -492,6 +492,24 @@
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"flake-utils_7": {
|
||||
"inputs": {
|
||||
"systems": "systems_7"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1710146030,
|
||||
"narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"gitignore": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
@ -700,16 +718,16 @@
|
||||
},
|
||||
"nixpkgs_6": {
|
||||
"locked": {
|
||||
"lastModified": 1723912943,
|
||||
"narHash": "sha256-39F9GzyhxYcY3wTeKuEFWRJWcrGBosO4nf4xzMTWZX8=",
|
||||
"owner": "danieldk",
|
||||
"lastModified": 1724915739,
|
||||
"narHash": "sha256-7PgRge4mn5akFvhPwefuaLQGbF5BnmxlwZJEf7CgbrE=",
|
||||
"owner": "nixos",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "b82cdca86dbb30013b76c4b55d48806476820a5c",
|
||||
"rev": "85be051bb60943d3328d91aaf2598798f87e19af",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "danieldk",
|
||||
"ref": "cuda-12.4",
|
||||
"owner": "nixos",
|
||||
"ref": "nixos-unstable-small",
|
||||
"repo": "nixpkgs",
|
||||
"type": "github"
|
||||
}
|
||||
@ -835,11 +853,11 @@
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1724638882,
|
||||
"narHash": "sha256-ap2jIQi/FuUHR6HCht6ASWhoz8EiB99XmI8Esot38VE=",
|
||||
"lastModified": 1725848835,
|
||||
"narHash": "sha256-u4lCr+tOEWhsFiww5G04U5jUNzaQJi0/ZMIDGiLeT14=",
|
||||
"owner": "oxalica",
|
||||
"repo": "rust-overlay",
|
||||
"rev": "19b70f147b9c67a759e35824b241f1ed92e46694",
|
||||
"rev": "2ef910a6276a2f34513d18f2f826a8dea72c3b3f",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@ -938,17 +956,33 @@
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"systems_7": {
|
||||
"locked": {
|
||||
"lastModified": 1681028828,
|
||||
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"tgi-nix": {
|
||||
"inputs": {
|
||||
"flake-compat": "flake-compat_4",
|
||||
"flake-utils": "flake-utils_7",
|
||||
"nixpkgs": "nixpkgs_6"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1725011596,
|
||||
"narHash": "sha256-zfq8lOXFgJnKxxsqSelHuKUvhxgH3cEmLoAgsOO62Cg=",
|
||||
"lastModified": 1725868835,
|
||||
"narHash": "sha256-6OFEaFFRCG/JKkU6kHV08EPEGM1MCuKZ70NlGJcL/JY=",
|
||||
"owner": "danieldk",
|
||||
"repo": "tgi-nix",
|
||||
"rev": "717c2b07e38538abf05237cca65b2d1363c2c9af",
|
||||
"rev": "87afbe21e2d2cc17e177c9965a64ba68ad7c22da",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
||||
@ -8,7 +8,7 @@
|
||||
"text": "\n"
|
||||
}
|
||||
],
|
||||
"created": 1725868995,
|
||||
"created": 1725874430,
|
||||
"id": "",
|
||||
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"object": "text_completion",
|
||||
@ -23,7 +23,7 @@
|
||||
"text": "\n"
|
||||
}
|
||||
],
|
||||
"created": 1725868995,
|
||||
"created": 1725874430,
|
||||
"id": "",
|
||||
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"object": "text_completion",
|
||||
@ -38,7 +38,7 @@
|
||||
"text": "\n"
|
||||
}
|
||||
],
|
||||
"created": 1725868995,
|
||||
"created": 1725874430,
|
||||
"id": "",
|
||||
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"object": "text_completion",
|
||||
@ -53,7 +53,7 @@
|
||||
"text": "hd"
|
||||
}
|
||||
],
|
||||
"created": 1725868995,
|
||||
"created": 1725874430,
|
||||
"id": "",
|
||||
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"object": "text_completion",
|
||||
@ -68,7 +68,7 @@
|
||||
"text": "What"
|
||||
}
|
||||
],
|
||||
"created": 1725868995,
|
||||
"created": 1725874430,
|
||||
"id": "",
|
||||
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"object": "text_completion",
|
||||
@ -80,10 +80,10 @@
|
||||
"finish_reason": "",
|
||||
"index": 1,
|
||||
"logprobs": null,
|
||||
"text": "Cache"
|
||||
"text": "rig"
|
||||
}
|
||||
],
|
||||
"created": 1725868995,
|
||||
"created": 1725874430,
|
||||
"id": "",
|
||||
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"object": "text_completion",
|
||||
@ -95,10 +95,10 @@
|
||||
"finish_reason": "",
|
||||
"index": 2,
|
||||
"logprobs": null,
|
||||
"text": "hd"
|
||||
"text": "\n"
|
||||
}
|
||||
],
|
||||
"created": 1725868995,
|
||||
"created": 1725874430,
|
||||
"id": "",
|
||||
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"object": "text_completion",
|
||||
@ -110,10 +110,10 @@
|
||||
"finish_reason": "",
|
||||
"index": 3,
|
||||
"logprobs": null,
|
||||
"text": ":"
|
||||
"text": "\n"
|
||||
}
|
||||
],
|
||||
"created": 1725868995,
|
||||
"created": 1725874430,
|
||||
"id": "",
|
||||
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"object": "text_completion",
|
||||
@ -125,40 +125,40 @@
|
||||
"finish_reason": "",
|
||||
"index": 0,
|
||||
"logprobs": null,
|
||||
"text": " Mil"
|
||||
}
|
||||
],
|
||||
"created": 1725868995,
|
||||
"id": "",
|
||||
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"object": "text_completion",
|
||||
"system_fingerprint": "2.2.1-dev0-native"
|
||||
},
|
||||
{
|
||||
"choices": [
|
||||
{
|
||||
"finish_reason": "",
|
||||
"index": 1,
|
||||
"logprobs": null,
|
||||
"text": "2"
|
||||
}
|
||||
],
|
||||
"created": 1725868995,
|
||||
"id": "",
|
||||
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"object": "text_completion",
|
||||
"system_fingerprint": "2.2.1-dev0-native"
|
||||
},
|
||||
{
|
||||
"choices": [
|
||||
{
|
||||
"finish_reason": "",
|
||||
"index": 2,
|
||||
"logprobs": null,
|
||||
"text": " Business"
|
||||
}
|
||||
],
|
||||
"created": 1725868995,
|
||||
"created": 1725874430,
|
||||
"id": "",
|
||||
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"object": "text_completion",
|
||||
"system_fingerprint": "2.2.1-dev0-native"
|
||||
},
|
||||
{
|
||||
"choices": [
|
||||
{
|
||||
"finish_reason": "",
|
||||
"index": 1,
|
||||
"logprobs": null,
|
||||
"text": " Business"
|
||||
}
|
||||
],
|
||||
"created": 1725874430,
|
||||
"id": "",
|
||||
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"object": "text_completion",
|
||||
"system_fingerprint": "2.2.1-dev0-native"
|
||||
},
|
||||
{
|
||||
"choices": [
|
||||
{
|
||||
"finish_reason": "",
|
||||
"index": 2,
|
||||
"logprobs": null,
|
||||
"text": "2"
|
||||
}
|
||||
],
|
||||
"created": 1725874430,
|
||||
"id": "",
|
||||
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"object": "text_completion",
|
||||
@ -173,7 +173,7 @@
|
||||
"text": "2"
|
||||
}
|
||||
],
|
||||
"created": 1725868995,
|
||||
"created": 1725874430,
|
||||
"id": "",
|
||||
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"object": "text_completion",
|
||||
@ -185,10 +185,10 @@
|
||||
"finish_reason": "",
|
||||
"index": 0,
|
||||
"logprobs": null,
|
||||
"text": "k"
|
||||
"text": ":"
|
||||
}
|
||||
],
|
||||
"created": 1725868995,
|
||||
"created": 1725874430,
|
||||
"id": "",
|
||||
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"object": "text_completion",
|
||||
@ -200,10 +200,10 @@
|
||||
"finish_reason": "",
|
||||
"index": 1,
|
||||
"logprobs": null,
|
||||
"text": "."
|
||||
"text": " Process"
|
||||
}
|
||||
],
|
||||
"created": 1725868995,
|
||||
"created": 1725874430,
|
||||
"id": "",
|
||||
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"object": "text_completion",
|
||||
@ -215,10 +215,10 @@
|
||||
"finish_reason": "",
|
||||
"index": 2,
|
||||
"logprobs": null,
|
||||
"text": "|"
|
||||
"text": "0"
|
||||
}
|
||||
],
|
||||
"created": 1725868995,
|
||||
"created": 1725874430,
|
||||
"id": "",
|
||||
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"object": "text_completion",
|
||||
@ -230,10 +230,10 @@
|
||||
"finish_reason": "",
|
||||
"index": 3,
|
||||
"logprobs": null,
|
||||
"text": "|"
|
||||
"text": "0"
|
||||
}
|
||||
],
|
||||
"created": 1725868995,
|
||||
"created": 1725874430,
|
||||
"id": "",
|
||||
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"object": "text_completion",
|
||||
@ -248,7 +248,7 @@
|
||||
"text": " And"
|
||||
}
|
||||
],
|
||||
"created": 1725868995,
|
||||
"created": 1725874430,
|
||||
"id": "",
|
||||
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"object": "text_completion",
|
||||
@ -263,7 +263,7 @@
|
||||
"text": " And"
|
||||
}
|
||||
],
|
||||
"created": 1725868995,
|
||||
"created": 1725874430,
|
||||
"id": "",
|
||||
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"object": "text_completion",
|
||||
@ -275,10 +275,10 @@
|
||||
"finish_reason": "",
|
||||
"index": 2,
|
||||
"logprobs": null,
|
||||
"text": " And"
|
||||
"text": "2"
|
||||
}
|
||||
],
|
||||
"created": 1725868995,
|
||||
"created": 1725874430,
|
||||
"id": "",
|
||||
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"object": "text_completion",
|
||||
@ -290,10 +290,10 @@
|
||||
"finish_reason": "",
|
||||
"index": 3,
|
||||
"logprobs": null,
|
||||
"text": "i"
|
||||
"text": "2"
|
||||
}
|
||||
],
|
||||
"created": 1725868995,
|
||||
"created": 1725874430,
|
||||
"id": "",
|
||||
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"object": "text_completion",
|
||||
@ -308,7 +308,7 @@
|
||||
"text": " Stock"
|
||||
}
|
||||
],
|
||||
"created": 1725868995,
|
||||
"created": 1725874430,
|
||||
"id": "",
|
||||
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"object": "text_completion",
|
||||
@ -320,10 +320,10 @@
|
||||
"finish_reason": "",
|
||||
"index": 1,
|
||||
"logprobs": null,
|
||||
"text": " How"
|
||||
"text": " Stock"
|
||||
}
|
||||
],
|
||||
"created": 1725868995,
|
||||
"created": 1725874430,
|
||||
"id": "",
|
||||
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"object": "text_completion",
|
||||
@ -335,10 +335,10 @@
|
||||
"finish_reason": "",
|
||||
"index": 2,
|
||||
"logprobs": null,
|
||||
"text": " i"
|
||||
"text": " Stock"
|
||||
}
|
||||
],
|
||||
"created": 1725868995,
|
||||
"created": 1725874430,
|
||||
"id": "",
|
||||
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"object": "text_completion",
|
||||
@ -350,10 +350,10 @@
|
||||
"finish_reason": "",
|
||||
"index": 3,
|
||||
"logprobs": null,
|
||||
"text": "'"
|
||||
"text": "0"
|
||||
}
|
||||
],
|
||||
"created": 1725868995,
|
||||
"created": 1725874430,
|
||||
"id": "",
|
||||
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"object": "text_completion",
|
||||
@ -365,10 +365,10 @@
|
||||
"finish_reason": "",
|
||||
"index": 0,
|
||||
"logprobs": null,
|
||||
"text": " U"
|
||||
"text": " Moh"
|
||||
}
|
||||
],
|
||||
"created": 1725868995,
|
||||
"created": 1725874431,
|
||||
"id": "",
|
||||
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"object": "text_completion",
|
||||
@ -380,10 +380,10 @@
|
||||
"finish_reason": "",
|
||||
"index": 1,
|
||||
"logprobs": null,
|
||||
"text": " often"
|
||||
"text": " ,"
|
||||
}
|
||||
],
|
||||
"created": 1725868995,
|
||||
"created": 1725874431,
|
||||
"id": "",
|
||||
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"object": "text_completion",
|
||||
@ -395,10 +395,10 @@
|
||||
"finish_reason": "",
|
||||
"index": 2,
|
||||
"logprobs": null,
|
||||
"text": "|"
|
||||
"text": " Moh"
|
||||
}
|
||||
],
|
||||
"created": 1725868995,
|
||||
"created": 1725874431,
|
||||
"id": "",
|
||||
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"object": "text_completion",
|
||||
@ -410,25 +410,25 @@
|
||||
"finish_reason": "",
|
||||
"index": 3,
|
||||
"logprobs": null,
|
||||
"text": "7"
|
||||
}
|
||||
],
|
||||
"created": 1725874431,
|
||||
"id": "",
|
||||
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"object": "text_completion",
|
||||
"system_fingerprint": "2.2.1-dev0-native"
|
||||
},
|
||||
{
|
||||
"choices": [
|
||||
{
|
||||
"finish_reason": "",
|
||||
"index": 0,
|
||||
"logprobs": null,
|
||||
"text": "s"
|
||||
}
|
||||
],
|
||||
"created": 1725868995,
|
||||
"id": "",
|
||||
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"object": "text_completion",
|
||||
"system_fingerprint": "2.2.1-dev0-native"
|
||||
},
|
||||
{
|
||||
"choices": [
|
||||
{
|
||||
"finish_reason": "",
|
||||
"index": 0,
|
||||
"logprobs": null,
|
||||
"text": "sa"
|
||||
}
|
||||
],
|
||||
"created": 1725868995,
|
||||
"created": 1725874431,
|
||||
"id": "",
|
||||
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"object": "text_completion",
|
||||
@ -440,10 +440,10 @@
|
||||
"finish_reason": "",
|
||||
"index": 1,
|
||||
"logprobs": null,
|
||||
"text": " has"
|
||||
"text": "s"
|
||||
}
|
||||
],
|
||||
"created": 1725868995,
|
||||
"created": 1725874431,
|
||||
"id": "",
|
||||
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"object": "text_completion",
|
||||
@ -458,7 +458,7 @@
|
||||
"text": "s"
|
||||
}
|
||||
],
|
||||
"created": 1725868995,
|
||||
"created": 1725874431,
|
||||
"id": "",
|
||||
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"object": "text_completion",
|
||||
@ -470,10 +470,10 @@
|
||||
"finish_reason": "",
|
||||
"index": 3,
|
||||
"logprobs": null,
|
||||
"text": " has"
|
||||
"text": "\n"
|
||||
}
|
||||
],
|
||||
"created": 1725868995,
|
||||
"created": 1725874431,
|
||||
"id": "",
|
||||
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"object": "text_completion",
|
||||
@ -485,10 +485,10 @@
|
||||
"finish_reason": "",
|
||||
"index": 0,
|
||||
"logprobs": null,
|
||||
"text": "?"
|
||||
"text": "`"
|
||||
}
|
||||
],
|
||||
"created": 1725868995,
|
||||
"created": 1725874431,
|
||||
"id": "",
|
||||
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"object": "text_completion",
|
||||
@ -500,10 +500,10 @@
|
||||
"finish_reason": "",
|
||||
"index": 1,
|
||||
"logprobs": null,
|
||||
"text": " a"
|
||||
"text": ","
|
||||
}
|
||||
],
|
||||
"created": 1725868995,
|
||||
"created": 1725874431,
|
||||
"id": "",
|
||||
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"object": "text_completion",
|
||||
@ -518,7 +518,7 @@
|
||||
"text": " a"
|
||||
}
|
||||
],
|
||||
"created": 1725868995,
|
||||
"created": 1725874431,
|
||||
"id": "",
|
||||
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"object": "text_completion",
|
||||
@ -530,10 +530,10 @@
|
||||
"finish_reason": "",
|
||||
"index": 3,
|
||||
"logprobs": null,
|
||||
"text": " a"
|
||||
"text": "R"
|
||||
}
|
||||
],
|
||||
"created": 1725868995,
|
||||
"created": 1725874431,
|
||||
"id": "",
|
||||
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"object": "text_completion",
|
||||
@ -545,10 +545,10 @@
|
||||
"finish_reason": "length",
|
||||
"index": 0,
|
||||
"logprobs": null,
|
||||
"text": "</s>"
|
||||
"text": "('\\"
|
||||
}
|
||||
],
|
||||
"created": 1725868995,
|
||||
"created": 1725874431,
|
||||
"id": "",
|
||||
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"object": "text_completion",
|
||||
@ -560,10 +560,10 @@
|
||||
"finish_reason": "length",
|
||||
"index": 1,
|
||||
"logprobs": null,
|
||||
"text": " sc"
|
||||
"text": " And"
|
||||
}
|
||||
],
|
||||
"created": 1725868995,
|
||||
"created": 1725874431,
|
||||
"id": "",
|
||||
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"object": "text_completion",
|
||||
@ -575,10 +575,10 @@
|
||||
"finish_reason": "length",
|
||||
"index": 2,
|
||||
"logprobs": null,
|
||||
"text": "|"
|
||||
"text": " Service"
|
||||
}
|
||||
],
|
||||
"created": 1725868995,
|
||||
"created": 1725874431,
|
||||
"id": "",
|
||||
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"object": "text_completion",
|
||||
@ -590,10 +590,10 @@
|
||||
"finish_reason": "length",
|
||||
"index": 3,
|
||||
"logprobs": null,
|
||||
"text": " sc"
|
||||
"text": "1"
|
||||
}
|
||||
],
|
||||
"created": 1725868995,
|
||||
"created": 1725874431,
|
||||
"id": "",
|
||||
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"object": "text_completion",
|
||||
|
||||
@ -4,14 +4,14 @@
|
||||
"finish_reason": "length",
|
||||
"index": 0,
|
||||
"logprobs": null,
|
||||
"text": " PR for flake8"
|
||||
"text": "\n2.2 How"
|
||||
}
|
||||
],
|
||||
"created": 1713284454,
|
||||
"created": 1725874238,
|
||||
"id": "",
|
||||
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"object": "text_completion",
|
||||
"system_fingerprint": "2.0.1-native",
|
||||
"system_fingerprint": "2.2.1-dev0-native",
|
||||
"usage": {
|
||||
"completion_tokens": 5,
|
||||
"prompt_tokens": 6,
|
||||
|
||||
@ -34,7 +34,7 @@ def test_flash_llama_completion_single_prompt(
|
||||
f"{flash_llama_completion.base_url}/v1/completions",
|
||||
json={
|
||||
"model": "tgi",
|
||||
"prompt": "Say this is a test",
|
||||
"prompt": "What is Deep Learning?",
|
||||
"max_tokens": 5,
|
||||
"seed": 0,
|
||||
},
|
||||
@ -43,7 +43,7 @@ def test_flash_llama_completion_single_prompt(
|
||||
)
|
||||
response = response.json()
|
||||
assert len(response["choices"]) == 1
|
||||
|
||||
assert response["choices"][0]["text"] == "\n2.2 How"
|
||||
assert response == response_snapshot
|
||||
|
||||
|
||||
@ -77,7 +77,7 @@ async def test_flash_llama_completion_many_prompts_stream(
|
||||
request = {
|
||||
"model": "tgi",
|
||||
"prompt": [
|
||||
"What color is the sky?",
|
||||
"What is Deep Learning?",
|
||||
"Is water wet?",
|
||||
"What is the capital of France?",
|
||||
"def mai",
|
||||
@ -90,6 +90,7 @@ async def test_flash_llama_completion_many_prompts_stream(
|
||||
url = f"{flash_llama_completion.base_url}/v1/completions"
|
||||
|
||||
chunks = []
|
||||
strings = [""] * 4
|
||||
async with ClientSession(headers=flash_llama_completion.headers) as session:
|
||||
async with session.post(url, json=request) as response:
|
||||
# iterate over the stream
|
||||
@ -108,7 +109,10 @@ async def test_flash_llama_completion_many_prompts_stream(
|
||||
for c in chunk:
|
||||
chunks.append(Completion(**c))
|
||||
assert "choices" in c
|
||||
assert 0 <= c["choices"][0]["index"] <= 4
|
||||
index = c["choices"][0]["index"]
|
||||
assert 0 <= index <= 4
|
||||
strings[index] += c["choices"][0]["text"]
|
||||
|
||||
assert response.status == 200
|
||||
# assert strings == ["What Business: And Stock Mohs`('\\", '\nrig Business Process And Stock ,s, And', '\n\n202 Stock Mohs a Service', 'hd\n20207\nR1']
|
||||
assert chunks == response_snapshot
|
||||
|
||||
1349
integration-tests/poetry.lock
generated
1349
integration-tests/poetry.lock
generated
File diff suppressed because it is too large
Load Diff
@ -12,3 +12,4 @@ text-generation = "^0.6.0"
|
||||
pytest = "^7.4.0"
|
||||
pytest-asyncio = "^0.21.1"
|
||||
docker = "^6.1.3"
|
||||
numpy = "^1.20"
|
||||
|
||||
@ -1,34 +1,36 @@
|
||||
aiohttp==3.8.5 ; python_version >= "3.10" and python_version < "3.13"
|
||||
aiohappyeyeballs==2.4.0 ; python_version >= "3.10" and python_version < "3.13"
|
||||
aiohttp==3.10.5 ; python_version >= "3.10" and python_version < "3.13"
|
||||
aiosignal==1.3.1 ; python_version >= "3.10" and python_version < "3.13"
|
||||
annotated-types==0.6.0 ; python_version >= "3.10" and python_version < "3.13"
|
||||
async-timeout==4.0.3 ; python_version >= "3.10" and python_version < "3.13"
|
||||
attrs==23.1.0 ; python_version >= "3.10" and python_version < "3.13"
|
||||
certifi==2023.7.22 ; python_version >= "3.10" and python_version < "3.13"
|
||||
charset-normalizer==3.2.0 ; python_version >= "3.10" and python_version < "3.13"
|
||||
annotated-types==0.7.0 ; python_version >= "3.10" and python_version < "3.13"
|
||||
async-timeout==4.0.3 ; python_version >= "3.10" and python_version < "3.11"
|
||||
attrs==24.2.0 ; python_version >= "3.10" and python_version < "3.13"
|
||||
certifi==2024.8.30 ; python_version >= "3.10" and python_version < "3.13"
|
||||
charset-normalizer==3.3.2 ; python_version >= "3.10" and python_version < "3.13"
|
||||
colorama==0.4.6 ; python_version >= "3.10" and python_version < "3.13" and (sys_platform == "win32" or platform_system == "Windows")
|
||||
docker==6.1.3 ; python_version >= "3.10" and python_version < "3.13"
|
||||
exceptiongroup==1.1.3 ; python_version >= "3.10" and python_version < "3.11"
|
||||
filelock==3.12.3 ; python_version >= "3.10" and python_version < "3.13"
|
||||
frozenlist==1.4.0 ; python_version >= "3.10" and python_version < "3.13"
|
||||
fsspec==2023.6.0 ; python_version >= "3.10" and python_version < "3.13"
|
||||
huggingface-hub==0.16.4 ; python_version >= "3.10" and python_version < "3.13"
|
||||
idna==3.4 ; python_version >= "3.10" and python_version < "3.13"
|
||||
exceptiongroup==1.2.2 ; python_version >= "3.10" and python_version < "3.11"
|
||||
filelock==3.16.0 ; python_version >= "3.10" and python_version < "3.13"
|
||||
frozenlist==1.4.1 ; python_version >= "3.10" and python_version < "3.13"
|
||||
fsspec==2024.9.0 ; python_version >= "3.10" and python_version < "3.13"
|
||||
huggingface-hub==0.24.6 ; python_version >= "3.10" and python_version < "3.13"
|
||||
idna==3.8 ; python_version >= "3.10" and python_version < "3.13"
|
||||
iniconfig==2.0.0 ; python_version >= "3.10" and python_version < "3.13"
|
||||
multidict==6.0.4 ; python_version >= "3.10" and python_version < "3.13"
|
||||
packaging==23.1 ; python_version >= "3.10" and python_version < "3.13"
|
||||
pluggy==1.3.0 ; python_version >= "3.10" and python_version < "3.13"
|
||||
pydantic-core==2.16.3 ; python_version >= "3.10" and python_version < "3.13"
|
||||
pydantic==2.6.4 ; python_version >= "3.10" and python_version < "3.13"
|
||||
pytest-asyncio==0.21.1 ; python_version >= "3.10" and python_version < "3.13"
|
||||
pytest==7.4.0 ; python_version >= "3.10" and python_version < "3.13"
|
||||
multidict==6.0.5 ; python_version >= "3.10" and python_version < "3.13"
|
||||
numpy==1.26.4 ; python_version >= "3.10" and python_version < "3.13"
|
||||
packaging==24.1 ; python_version >= "3.10" and python_version < "3.13"
|
||||
pluggy==1.5.0 ; python_version >= "3.10" and python_version < "3.13"
|
||||
pydantic-core==2.23.3 ; python_version >= "3.10" and python_version < "3.13"
|
||||
pydantic==2.9.1 ; python_version >= "3.10" and python_version < "3.13"
|
||||
pytest-asyncio==0.21.2 ; python_version >= "3.10" and python_version < "3.13"
|
||||
pytest==7.4.4 ; python_version >= "3.10" and python_version < "3.13"
|
||||
pywin32==306 ; python_version >= "3.10" and python_version < "3.13" and sys_platform == "win32"
|
||||
pyyaml==6.0.1 ; python_version >= "3.10" and python_version < "3.13"
|
||||
requests==2.31.0 ; python_version >= "3.10" and python_version < "3.13"
|
||||
pyyaml==6.0.2 ; python_version >= "3.10" and python_version < "3.13"
|
||||
requests==2.32.3 ; python_version >= "3.10" and python_version < "3.13"
|
||||
syrupy==4.7.1 ; python_version >= "3.10" and python_version < "3.13"
|
||||
text-generation==0.6.1 ; python_version >= "3.10" and python_version < "3.13"
|
||||
tomli==2.0.1 ; python_version >= "3.10" and python_version < "3.11"
|
||||
tqdm==4.66.1 ; python_version >= "3.10" and python_version < "3.13"
|
||||
typing-extensions==4.7.1 ; python_version >= "3.10" and python_version < "3.13"
|
||||
urllib3==2.0.4 ; python_version >= "3.10" and python_version < "3.13"
|
||||
websocket-client==1.6.2 ; python_version >= "3.10" and python_version < "3.13"
|
||||
yarl==1.9.2 ; python_version >= "3.10" and python_version < "3.13"
|
||||
tqdm==4.66.5 ; python_version >= "3.10" and python_version < "3.13"
|
||||
typing-extensions==4.12.2 ; python_version >= "3.10" and python_version < "3.13"
|
||||
urllib3==2.2.2 ; python_version >= "3.10" and python_version < "3.13"
|
||||
websocket-client==1.8.0 ; python_version >= "3.10" and python_version < "3.13"
|
||||
yarl==1.11.0 ; python_version >= "3.10" and python_version < "3.13"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user