mirror of
https://github.com/huggingface/text-generation-inference.git
synced 2025-07-27 02:10:17 +00:00
wip
This commit is contained in:
OlivierDehaene
parent
4dfa6fbb62
commit
a28a8ebdb5
571
Cargo.lock
generated
571
Cargo.lock
generated
@ -28,15 +28,6 @@ dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "android_system_properties"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ansi_term"
|
||||
version = "0.12.1"
|
||||
@ -102,6 +93,17 @@ version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
|
||||
|
||||
[[package]]
|
||||
name = "average"
|
||||
version = "0.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "843ec791d3f24503bbf72bbd5e49a3ab4dbb4bcd0a8ef6b0c908efa73caa27b1"
|
||||
dependencies = [
|
||||
"easy-cast",
|
||||
"float-ord",
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "axum"
|
||||
version = "0.6.9"
|
||||
@ -201,12 +203,6 @@ version = "3.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535"
|
||||
|
||||
[[package]]
|
||||
name = "bytemuck"
|
||||
version = "1.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "17febce684fd15d89027105661fec94afb475cb995fbc59d2865198446ba2eea"
|
||||
|
||||
[[package]]
|
||||
name = "byteorder"
|
||||
version = "1.4.3"
|
||||
@ -263,6 +259,12 @@ dependencies = [
|
||||
"zip-extensions",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cassowary"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53"
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.0.79"
|
||||
@ -275,21 +277,6 @@ version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
|
||||
|
||||
[[package]]
|
||||
name = "chrono"
|
||||
version = "0.4.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f"
|
||||
dependencies = [
|
||||
"iana-time-zone",
|
||||
"js-sys",
|
||||
"num-integer",
|
||||
"num-traits",
|
||||
"time",
|
||||
"wasm-bindgen",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "2.34.0"
|
||||
@ -342,31 +329,6 @@ dependencies = [
|
||||
"os_str_bytes",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cmake"
|
||||
version = "0.1.49"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "db34956e100b30725f2eb215f90d4871051239535632f84fea3bc92722c66b7c"
|
||||
dependencies = [
|
||||
"cc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "codespan-reporting"
|
||||
version = "0.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e"
|
||||
dependencies = [
|
||||
"termcolor",
|
||||
"unicode-width",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "color_quant"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b"
|
||||
|
||||
[[package]]
|
||||
name = "console"
|
||||
version = "0.15.5"
|
||||
@ -380,12 +342,6 @@ dependencies = [
|
||||
"windows-sys 0.42.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "const-cstr"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed3d0b5ff30645a68f35ece8cea4556ca14ef8a1651455f789a099a0513532a6"
|
||||
|
||||
[[package]]
|
||||
name = "core-foundation"
|
||||
version = "0.9.3"
|
||||
@ -402,43 +358,6 @@ version = "0.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc"
|
||||
|
||||
[[package]]
|
||||
name = "core-graphics"
|
||||
version = "0.22.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2581bbab3b8ffc6fcbd550bf46c355135d16e9ff2a6ea032ad6b9bf1d7efe4fb"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"core-foundation",
|
||||
"core-graphics-types",
|
||||
"foreign-types",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "core-graphics-types"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3a68b68b3446082644c91ac778bf50cd4104bfb002b5a6a7c44cca5a2c70788b"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"core-foundation",
|
||||
"foreign-types",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "core-text"
|
||||
version = "19.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "99d74ada66e07c1cefa18f8abfba765b486f250de2e4a999e5727fc0dd4b4a25"
|
||||
dependencies = [
|
||||
"core-foundation",
|
||||
"core-graphics",
|
||||
"foreign-types",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cpufeatures"
|
||||
version = "0.2.5"
|
||||
@ -500,6 +419,31 @@ dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossterm"
|
||||
version = "0.26.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a84cda67535339806297f1b331d6dd6320470d2a0fe65381e79ee9e156dd3d13"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"crossterm_winapi",
|
||||
"libc",
|
||||
"mio",
|
||||
"parking_lot",
|
||||
"signal-hook",
|
||||
"signal-hook-mio",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossterm_winapi"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2ae1b35a484aa10e07fe0638d02301c5ad24de82d310ccbd2f3693da5f09bf1c"
|
||||
dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crypto-common"
|
||||
version = "0.1.6"
|
||||
@ -520,50 +464,6 @@ dependencies = [
|
||||
"windows-sys 0.45.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cxx"
|
||||
version = "1.0.92"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9a140f260e6f3f79013b8bfc65e7ce630c9ab4388c6a89c71e07226f49487b72"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"cxxbridge-flags",
|
||||
"cxxbridge-macro",
|
||||
"link-cplusplus",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cxx-build"
|
||||
version = "1.0.92"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "da6383f459341ea689374bf0a42979739dc421874f112ff26f829b8040b8e613"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"codespan-reporting",
|
||||
"once_cell",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"scratch",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cxxbridge-flags"
|
||||
version = "1.0.92"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "90201c1a650e95ccff1c8c0bb5a343213bdd317c6e600a93075bca2eff54ec97"
|
||||
|
||||
[[package]]
|
||||
name = "cxxbridge-macro"
|
||||
version = "1.0.92"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b75aed41bb2e6367cae39e6326ef817a851db13c13e4f3263714ca3cfb8de56"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "darling"
|
||||
version = "0.10.2"
|
||||
@ -665,16 +565,6 @@ dependencies = [
|
||||
"dirs-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dirs-next"
|
||||
version = "2.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"dirs-sys-next",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dirs-sys"
|
||||
version = "0.3.7"
|
||||
@ -687,35 +577,12 @@ dependencies = [
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dirs-sys-next"
|
||||
version = "0.1.2"
|
||||
name = "easy-cast"
|
||||
version = "0.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d"
|
||||
checksum = "4bd102ee8c418348759919b83b81cdbdc933ffe29740b903df448b4bafaa348e"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"redox_users",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dlib"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac1b7517328c04c2aa68422fc60a41b92208182142ed04a25879c26c8f878794"
|
||||
dependencies = [
|
||||
"libloading",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dwrote"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "439a1c2ba5611ad3ed731280541d36d2e9c4ac5e7fb818a27b604bdc5a6aa65b"
|
||||
dependencies = [
|
||||
"lazy_static",
|
||||
"libc",
|
||||
"winapi",
|
||||
"wio",
|
||||
"libm",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -808,9 +675,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "float-ord"
|
||||
version = "0.2.0"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7bad48618fdb549078c333a7a8528acb57af271d0433bdecd523eb620628364e"
|
||||
checksum = "8ce81f49ae8a0482e4c55ea62ebbd7e5a686af544c00b9d090bba3ff9be97b3d"
|
||||
|
||||
[[package]]
|
||||
name = "float_eq"
|
||||
@ -824,31 +691,6 @@ version = "1.0.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
|
||||
|
||||
[[package]]
|
||||
name = "font-kit"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "21fe28504d371085fae9ac7a3450f0b289ab71e07c8e57baa3fb68b9e57d6ce5"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"byteorder",
|
||||
"core-foundation",
|
||||
"core-graphics",
|
||||
"core-text",
|
||||
"dirs-next",
|
||||
"dwrote",
|
||||
"float-ord",
|
||||
"freetype",
|
||||
"lazy_static",
|
||||
"libc",
|
||||
"log",
|
||||
"pathfinder_geometry",
|
||||
"pathfinder_simd",
|
||||
"walkdir",
|
||||
"winapi",
|
||||
"yeslogic-fontconfig-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "foreign-types"
|
||||
version = "0.3.2"
|
||||
@ -873,27 +715,6 @@ dependencies = [
|
||||
"percent-encoding",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "freetype"
|
||||
version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bee38378a9e3db1cc693b4f88d166ae375338a0ff75cb8263e1c601d51f35dc6"
|
||||
dependencies = [
|
||||
"freetype-sys",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "freetype-sys"
|
||||
version = "0.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a37d4011c0cc628dfa766fcc195454f4b068d7afdc2adfd28861191d866e731a"
|
||||
dependencies = [
|
||||
"cmake",
|
||||
"libc",
|
||||
"pkg-config",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fs2"
|
||||
version = "0.4.3"
|
||||
@ -1014,16 +835,6 @@ dependencies = [
|
||||
"wasi 0.11.0+wasi-snapshot-preview1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "gif"
|
||||
version = "0.11.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3edd93c6756b4dfaf2709eafcc345ba2636565295c198a9cfbf75fa5e3e00b06"
|
||||
dependencies = [
|
||||
"color_quant",
|
||||
"weezl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "glob"
|
||||
version = "0.3.1"
|
||||
@ -1187,30 +998,6 @@ dependencies = [
|
||||
"tokio-native-tls",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "iana-time-zone"
|
||||
version = "0.1.53"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765"
|
||||
dependencies = [
|
||||
"android_system_properties",
|
||||
"core-foundation-sys",
|
||||
"iana-time-zone-haiku",
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "iana-time-zone-haiku"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca"
|
||||
dependencies = [
|
||||
"cxx",
|
||||
"cxx-build",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ident_case"
|
||||
version = "1.0.1"
|
||||
@ -1227,21 +1014,6 @@ dependencies = [
|
||||
"unicode-normalization",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "image"
|
||||
version = "0.24.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "69b7ea949b537b0fd0af141fff8c77690f2ce96f4f41f042ccb6c69c6c965945"
|
||||
dependencies = [
|
||||
"bytemuck",
|
||||
"byteorder",
|
||||
"color_quant",
|
||||
"jpeg-decoder",
|
||||
"num-rational",
|
||||
"num-traits",
|
||||
"png",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "indexmap"
|
||||
version = "1.9.2"
|
||||
@ -1347,12 +1119,6 @@ version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440"
|
||||
|
||||
[[package]]
|
||||
name = "jpeg-decoder"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bc0000e42512c92e31c2252315bda326620a4e034105e900c98ec492fa077b3e"
|
||||
|
||||
[[package]]
|
||||
name = "js-sys"
|
||||
version = "0.3.61"
|
||||
@ -1375,23 +1141,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79"
|
||||
|
||||
[[package]]
|
||||
name = "libloading"
|
||||
version = "0.7.4"
|
||||
name = "libm"
|
||||
version = "0.2.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "link-cplusplus"
|
||||
version = "1.0.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5"
|
||||
dependencies = [
|
||||
"cc",
|
||||
]
|
||||
checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb"
|
||||
|
||||
[[package]]
|
||||
name = "linux-raw-sys"
|
||||
@ -1636,27 +1389,6 @@ dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-integer"
|
||||
version = "0.1.45"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-rational"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"num-integer",
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-traits"
|
||||
version = "0.2.15"
|
||||
@ -1664,6 +1396,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"libm",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -1882,41 +1615,12 @@ version = "1.0.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d01a5bd0424d00070b0098dd17ebca6f961a959dead1dbcbbbc1d1cd8d3deeba"
|
||||
|
||||
[[package]]
|
||||
name = "pathfinder_geometry"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b7b7e7b4ea703700ce73ebf128e1450eb69c3a8329199ffbfb9b2a0418e5ad3"
|
||||
dependencies = [
|
||||
"log",
|
||||
"pathfinder_simd",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pathfinder_simd"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "39fe46acc5503595e5949c17b818714d26fdf9b4920eacf3b2947f0199f4a6ff"
|
||||
dependencies = [
|
||||
"rustc_version",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "percent-encoding"
|
||||
version = "2.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e"
|
||||
|
||||
[[package]]
|
||||
name = "pest"
|
||||
version = "2.5.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8cbd939b234e95d72bc393d51788aec68aeeb5d51e748ca08ff3aad58cb722f7"
|
||||
dependencies = [
|
||||
"thiserror",
|
||||
"ucd-trie",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "petgraph"
|
||||
version = "0.6.3"
|
||||
@ -1965,64 +1669,6 @@ version = "0.3.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160"
|
||||
|
||||
[[package]]
|
||||
name = "plotters"
|
||||
version = "0.3.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2538b639e642295546c50fcd545198c9d64ee2a38620a628724a3b266d5fbf97"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"font-kit",
|
||||
"image",
|
||||
"lazy_static",
|
||||
"num-traits",
|
||||
"pathfinder_geometry",
|
||||
"plotters-backend",
|
||||
"plotters-bitmap",
|
||||
"plotters-svg",
|
||||
"ttf-parser",
|
||||
"wasm-bindgen",
|
||||
"web-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "plotters-backend"
|
||||
version = "0.3.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "193228616381fecdc1224c62e96946dfbc73ff4384fba576e052ff8c1bea8142"
|
||||
|
||||
[[package]]
|
||||
name = "plotters-bitmap"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0c4a1f21490a6cf4a84c272ad20bd7844ed99a3178187a4c5ab7f2051295beef"
|
||||
dependencies = [
|
||||
"gif",
|
||||
"image",
|
||||
"plotters-backend",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "plotters-svg"
|
||||
version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f9a81d2759aae1dae668f783c308bc5c8ebd191ff4184aaa1b37f65a6ae5a56f"
|
||||
dependencies = [
|
||||
"plotters-backend",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "png"
|
||||
version = "0.17.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5d708eaf860a19b19ce538740d2b4bdeeb8337fa53f7738455e706623ad5c638"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"crc32fast",
|
||||
"flate2",
|
||||
"miniz_oxide",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "portable-atomic"
|
||||
version = "0.3.19"
|
||||
@ -2187,6 +1833,19 @@ dependencies = [
|
||||
"getrandom",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ratatui"
|
||||
version = "0.20.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dcc0d032bccba900ee32151ec0265667535c230169f5a011154cdcd984e16829"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"cassowary",
|
||||
"crossterm",
|
||||
"unicode-segmentation",
|
||||
"unicode-width",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "raw-cpuid"
|
||||
version = "10.7.0"
|
||||
@ -2347,15 +2006,6 @@ dependencies = [
|
||||
"walkdir",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustc_version"
|
||||
version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f0dfe2087c51c460008730de8b57e6a320782fbfb312e1f4d520e6c6fae155ee"
|
||||
dependencies = [
|
||||
"semver",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustix"
|
||||
version = "0.36.8"
|
||||
@ -2406,12 +2056,6 @@ version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
|
||||
|
||||
[[package]]
|
||||
name = "scratch"
|
||||
version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1"
|
||||
|
||||
[[package]]
|
||||
name = "security-framework"
|
||||
version = "2.8.2"
|
||||
@ -2435,24 +2079,6 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "semver"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6"
|
||||
dependencies = [
|
||||
"semver-parser",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "semver-parser"
|
||||
version = "0.10.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "00b0bef5b7f9e0df16536d3961cfb6e84331c065b4066afb39768d0e319411f7"
|
||||
dependencies = [
|
||||
"pest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.152"
|
||||
@ -2534,6 +2160,27 @@ dependencies = [
|
||||
"dirs 4.0.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "signal-hook"
|
||||
version = "0.3.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "732768f1176d21d09e076c23a93123d40bba92d50c4058da34d45c8de8e682b9"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"signal-hook-registry",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "signal-hook-mio"
|
||||
version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"mio",
|
||||
"signal-hook",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "signal-hook-registry"
|
||||
version = "1.4.1"
|
||||
@ -2674,9 +2321,12 @@ dependencies = [
|
||||
name = "text-generation-benchmark"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"average",
|
||||
"clap 4.1.8",
|
||||
"itertools 0.10.5",
|
||||
"plotters",
|
||||
"crossterm",
|
||||
"ratatui",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"text-generation-client",
|
||||
"thiserror",
|
||||
"tokenizers",
|
||||
@ -3155,24 +2805,12 @@ version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed"
|
||||
|
||||
[[package]]
|
||||
name = "ttf-parser"
|
||||
version = "0.15.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7b3e06c9b9d80ed6b745c7159c40b311ad2916abb34a49e9be2653b90db0d8dd"
|
||||
|
||||
[[package]]
|
||||
name = "typenum"
|
||||
version = "1.16.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba"
|
||||
|
||||
[[package]]
|
||||
name = "ucd-trie"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9e79c4d996edb816c91e4308506774452e55e95c3c9de07b6729e17e15a5ef81"
|
||||
|
||||
[[package]]
|
||||
name = "unicase"
|
||||
version = "2.6.0"
|
||||
@ -3414,12 +3052,6 @@ dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "weezl"
|
||||
version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9193164d4de03a926d909d3bc7c30543cecb35400c02114792c2cae20d5e2dbb"
|
||||
|
||||
[[package]]
|
||||
name = "which"
|
||||
version = "4.4.0"
|
||||
@ -3552,15 +3184,6 @@ dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wio"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5d129932f4644ac2396cb456385cbf9e63b5b30c6e8dc4820bdca4eb082037a5"
|
||||
dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "xattr"
|
||||
version = "0.2.3"
|
||||
@ -3570,18 +3193,6 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "yeslogic-fontconfig-sys"
|
||||
version = "3.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f2bbd69036d397ebbff671b1b8e4d918610c181c5a16073b96f984a38d08c386"
|
||||
dependencies = [
|
||||
"const-cstr",
|
||||
"dlib",
|
||||
"once_cell",
|
||||
"pkg-config",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zip"
|
||||
version = "0.5.13"
|
||||
|
||||
@ -13,13 +13,16 @@ name = "text-generation-bench"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
average = "0.13"
|
||||
clap = { version = "4.1.4", features = ["derive", "env"] }
|
||||
itertools = "0.10.5"
|
||||
plotters = "0.3.4"
|
||||
crossterm = "0.26"
|
||||
serde = {version = "1.0.142", features = ["derive"]}
|
||||
serde_json = "1.0"
|
||||
text-generation-client = { path = "../router/client" }
|
||||
thiserror = "1.0.38"
|
||||
tokenizers = "0.13.2"
|
||||
tokio = { version = "1.25.0", features = ["rt", "rt-multi-thread", "parking_lot", "signal", "sync"] }
|
||||
tui = {package = "ratatui", version = "0.20", default-features = false, features = ["crossterm"]}
|
||||
tracing = "0.1.37"
|
||||
tracing-subscriber = { version = "0.3.16", features = ["json", "env-filter"] }
|
||||
|
||||
|
||||
@ -1,13 +1,14 @@
|
||||
mod ui;
|
||||
|
||||
use std::time::Duration;
|
||||
use tokenizers::{Tokenizer, TruncationDirection};
|
||||
use tokio::time;
|
||||
use text_generation_client::{ShardedClient, Request, Batch, StoppingCriteriaParameters, NextTokenChooserParameters};
|
||||
use time::Instant;
|
||||
use plotters::prelude::*;
|
||||
use itertools::Itertools;
|
||||
use tokio::sync::mpsc;
|
||||
use crate::ui::UI;
|
||||
|
||||
const LOREM_IPSUM: &str = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.";
|
||||
const OUT_FILE_NAME: &'static str = "errorbar.png";
|
||||
|
||||
enum Step {
|
||||
Prefill,
|
||||
@ -25,154 +26,107 @@ struct Run {
|
||||
pub async fn run(
|
||||
tokenizer: Tokenizer,
|
||||
batch_size: Vec<u32>,
|
||||
sequence_length: Vec<u32>,
|
||||
decode_length: Vec<u32>,
|
||||
runs: u32,
|
||||
mut client: ShardedClient,
|
||||
sequence_length: u32,
|
||||
decode_length: u32,
|
||||
runs: usize,
|
||||
// mut client: ShardedClient,
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
// let prefill_runs = benchmark_prefill(&tokenizer, &batch_size, &sequence_length, &decode_length, runs, &mut client).await;
|
||||
let mut runs: Vec<(f64, f64)> = Vec::new();
|
||||
for i in 0..10{
|
||||
for j in 0..10 {
|
||||
runs.push((i as f64, j as f64));
|
||||
}
|
||||
|
||||
let (sender, receiver) = mpsc::channel(8);
|
||||
|
||||
|
||||
tokio::spawn(
|
||||
UI {
|
||||
n_run: runs,
|
||||
n_batch: batch_size.len(),
|
||||
n_batch_done: 0,
|
||||
run_receiver: receiver,
|
||||
}.draw()
|
||||
);
|
||||
|
||||
|
||||
for n in 0..runs {
|
||||
sender.send(()).await.unwrap();
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
}
|
||||
|
||||
let data = runs;
|
||||
// let down_sampled = down_sample(&data[..]);
|
||||
|
||||
let root = BitMapBackend::new(OUT_FILE_NAME, (1024, 768)).into_drawing_area();
|
||||
|
||||
root.fill(&WHITE)?;
|
||||
|
||||
let mut chart = ChartBuilder::on(&root)
|
||||
.caption("Linear Function with Noise", ("sans-serif", 60))
|
||||
.margin(10)
|
||||
.set_label_area_size(LabelAreaPosition::Left, 40)
|
||||
.set_label_area_size(LabelAreaPosition::Bottom, 40)
|
||||
.build_cartesian_2d(-10f64..10f64, -10f64..10f64)?;
|
||||
|
||||
chart.configure_mesh().draw()?;
|
||||
|
||||
chart
|
||||
.draw_series(LineSeries::new(data, &GREEN.mix(0.3)))?
|
||||
.label("Raw Data");
|
||||
|
||||
// chart.draw_series(LineSeries::new(
|
||||
// down_sampled.iter().map(|(x, _, y, _)| (*x, *y)),
|
||||
// &BLUE,
|
||||
// ))?;
|
||||
|
||||
// chart
|
||||
// .draw_series(
|
||||
// down_sampled.iter().map(|(x, yl, ym, yh)| {
|
||||
// ErrorBar::new_vertical(*x, *yl, *ym, *yh, BLUE.filled(), 20)
|
||||
// }),
|
||||
// )?
|
||||
// .label("Down-sampled")
|
||||
// .legend(|(x, y)| PathElement::new(vec![(x, y), (x, y)], &BLUE));
|
||||
|
||||
chart
|
||||
.configure_series_labels()
|
||||
.background_style(WHITE.filled())
|
||||
.draw()?;
|
||||
|
||||
// To avoid the IO failure being ignored silently, we manually call the present function
|
||||
root.present().expect("Unable to write result to file, please make sure 'plotters-doc-data' dir exists under current dir");
|
||||
println!("Result has been saved to {}", OUT_FILE_NAME);
|
||||
Ok(())
|
||||
|
||||
}
|
||||
|
||||
// fn down_sample(data: &[(f64, f64)]) -> Vec<(f64, f64, f64, f64)> {
|
||||
// let down_sampled: Vec<_> = data
|
||||
// .iter()
|
||||
// .into_iter()
|
||||
// .map(|(x, g)| {
|
||||
// let mut g: Vec<_> = g.map(|(_, y)| *y).collect();
|
||||
// g.sort_by(|a, b| a.partial_cmp(b).unwrap());
|
||||
// (
|
||||
// x,
|
||||
// g[0],
|
||||
// g.iter().sum::<f64>() / g.len() as f64,
|
||||
// g[g.len() - 1],
|
||||
// )
|
||||
// })
|
||||
// .collect();
|
||||
// down_sampled
|
||||
//
|
||||
// async fn benchmark_prefill(tokenizer: &Tokenizer,
|
||||
// batch_size: &Vec<u32>,
|
||||
// sequence_length: u32,
|
||||
// decode_length: u32,
|
||||
// runs: u32,
|
||||
// client: &mut ShardedClient) -> Vec<Run> {
|
||||
// let mut results = Vec::new();
|
||||
//
|
||||
// let lorem_ipsum_length = tokenizer.encode(LOREM_IPSUM, true).unwrap().len();
|
||||
//
|
||||
// for s in sequence_length {
|
||||
// let sequence = create_sequence(s, lorem_ipsum_length, tokenizer);
|
||||
// for b in batch_size {
|
||||
// for d in decode_length {
|
||||
// let requests = (0..*b).map(|id| {
|
||||
// Request {
|
||||
// id: id.into(),
|
||||
// inputs: sequence.clone(),
|
||||
// input_length: *s,
|
||||
// parameters: Some(NextTokenChooserParameters {
|
||||
// temperature: 1.0,
|
||||
// top_k: 0,
|
||||
// top_p: 1.0,
|
||||
// typical_p: 1.0,
|
||||
// do_sample: false,
|
||||
// seed: 0,
|
||||
// repetition_penalty: 1.0,
|
||||
// watermark: false,
|
||||
// }),
|
||||
// stopping_parameters: Some(StoppingCriteriaParameters {
|
||||
// max_new_tokens: *d,
|
||||
// stop_sequences: vec![],
|
||||
// ignore_eos_token: true,
|
||||
// }),
|
||||
// }
|
||||
// }).collect();
|
||||
//
|
||||
// let batch = Batch {
|
||||
// id: 0,
|
||||
// requests,
|
||||
// size: *b,
|
||||
// };
|
||||
//
|
||||
// for _ in 0..runs {
|
||||
// let start_time = Instant::now();
|
||||
// client.prefill(batch.clone()).await.unwrap();
|
||||
// let elasped = start_time.elapsed();
|
||||
//
|
||||
// client.clear_cache().await.unwrap();
|
||||
//
|
||||
// results.push(Run {
|
||||
// step: Step::Prefill,
|
||||
// batch_size: *b,
|
||||
// sequence_length: *s,
|
||||
// decode_length: *d,
|
||||
// time: elasped,
|
||||
// });
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// results
|
||||
// }
|
||||
|
||||
async fn benchmark_prefill(tokenizer: &Tokenizer,
|
||||
batch_size: &Vec<u32>,
|
||||
sequence_length: &Vec<u32>,
|
||||
decode_length: &Vec<u32>,
|
||||
runs: u32,
|
||||
client: &mut ShardedClient) -> Vec<Run> {
|
||||
let mut results = Vec::new();
|
||||
|
||||
let lorem_ipsum_length = tokenizer.encode(LOREM_IPSUM, true).unwrap().len();
|
||||
|
||||
for s in sequence_length {
|
||||
let sequence = create_sequence(s, lorem_ipsum_length, tokenizer);
|
||||
for b in batch_size {
|
||||
for d in decode_length {
|
||||
let requests = (0..*b).map(|id| {
|
||||
Request {
|
||||
id: id.into(),
|
||||
inputs: sequence.clone(),
|
||||
input_length: *s,
|
||||
parameters: Some(NextTokenChooserParameters {
|
||||
temperature: 1.0,
|
||||
top_k: 0,
|
||||
top_p: 1.0,
|
||||
typical_p: 1.0,
|
||||
do_sample: false,
|
||||
seed: 0,
|
||||
repetition_penalty: 1.0,
|
||||
watermark: false,
|
||||
}),
|
||||
stopping_parameters: Some(StoppingCriteriaParameters {
|
||||
max_new_tokens: *d,
|
||||
stop_sequences: vec![],
|
||||
ignore_eos_token: true,
|
||||
}),
|
||||
}
|
||||
}).collect();
|
||||
|
||||
let batch = Batch {
|
||||
id: 0,
|
||||
requests,
|
||||
size: *b,
|
||||
};
|
||||
|
||||
for _ in 0..runs {
|
||||
let start_time = Instant::now();
|
||||
client.prefill(batch.clone()).await.unwrap();
|
||||
let elasped = start_time.elapsed();
|
||||
|
||||
client.clear_cache().await.unwrap();
|
||||
|
||||
results.push(Run {
|
||||
step: Step::Prefill,
|
||||
batch_size: *b,
|
||||
sequence_length: *s,
|
||||
decode_length: *d,
|
||||
time: elasped,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
results
|
||||
}
|
||||
|
||||
fn create_sequence(sequence_length: &u32, lorem_ipsum_length: usize, tokenizer: &Tokenizer) -> String {
|
||||
// Repeat lorem ipsum to cover sequence length
|
||||
let string_sequence = LOREM_IPSUM.repeat((0..*sequence_length).step_by(lorem_ipsum_length).len());
|
||||
// Encode sequence
|
||||
let mut encoding = tokenizer.encode(string_sequence, true).unwrap();
|
||||
// Truncate to sequence_length
|
||||
encoding.truncate(*sequence_length as usize, 0, TruncationDirection::Left);
|
||||
// Decode
|
||||
tokenizer.decode(Vec::from(encoding.get_ids()), false).unwrap()
|
||||
}
|
||||
//
|
||||
// fn create_sequence(sequence_length: &u32, lorem_ipsum_length: usize, tokenizer: &Tokenizer) -> String {
|
||||
// // Repeat lorem ipsum to cover sequence length
|
||||
// let string_sequence = LOREM_IPSUM.repeat((0..*sequence_length).step_by(lorem_ipsum_length).len());
|
||||
// // Encode sequence
|
||||
// let mut encoding = tokenizer.encode(string_sequence, true).unwrap();
|
||||
// // Truncate to sequence_length
|
||||
// encoding.truncate(*sequence_length as usize, 0, TruncationDirection::Left);
|
||||
// // Decode
|
||||
// tokenizer.decode(Vec::from(encoding.get_ids()), false).unwrap()
|
||||
// }
|
||||
@ -16,16 +16,16 @@ struct Args {
|
||||
#[clap(default_value = "1", long, env)]
|
||||
batch_size: Vec<u32>,
|
||||
#[clap(default_value = "128", long, env)]
|
||||
sequence_length: Vec<u32>,
|
||||
sequence_length: u32,
|
||||
#[clap(default_value = "100", long, env)]
|
||||
decode_length: Vec<u32>,
|
||||
#[clap(default_value = "1", long, env)]
|
||||
runs: u32,
|
||||
decode_length: u32,
|
||||
#[clap(default_value = "10", long, env)]
|
||||
runs: usize,
|
||||
#[clap(default_value = "/tmp/text-generation-0", long, env)]
|
||||
master_shard_uds_path: String,
|
||||
}
|
||||
|
||||
fn main() -> Result<(), std::io::Error> {
|
||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
// Get args
|
||||
let args = Args::parse();
|
||||
// Pattern match configuration
|
||||
@ -60,14 +60,14 @@ fn main() -> Result<(), std::io::Error> {
|
||||
init_logging();
|
||||
|
||||
// Instantiate sharded client from the master unix socket
|
||||
let mut sharded_client = ShardedClient::connect_uds(master_shard_uds_path)
|
||||
.await
|
||||
.expect("Could not connect to server");
|
||||
// let mut sharded_client = ShardedClient::connect_uds(master_shard_uds_path)
|
||||
// .await
|
||||
// .expect("Could not connect to server");
|
||||
// Clear the cache; useful if the webserver rebooted
|
||||
sharded_client
|
||||
.clear_cache()
|
||||
.await
|
||||
.expect("Unable to clear cache");
|
||||
// sharded_client
|
||||
// .clear_cache()
|
||||
// .await
|
||||
// .expect("Unable to clear cache");
|
||||
tracing::info!("Connected");
|
||||
|
||||
text_generation_benchmark::run(
|
||||
@ -76,8 +76,8 @@ fn main() -> Result<(), std::io::Error> {
|
||||
sequence_length,
|
||||
decode_length,
|
||||
runs,
|
||||
sharded_client,
|
||||
).await;
|
||||
// sharded_client,
|
||||
).await.unwrap();
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
|
||||
199
benchmark/src/ui.rs
Normal file
199
benchmark/src/ui.rs
Normal file
@ -0,0 +1,199 @@
|
||||
/// Inspired by https://github.com/hatoo/oha/blob/master/src/monitor.rs
|
||||
|
||||
use crossterm::event::{Event, KeyCode, KeyEvent, KeyModifiers};
|
||||
use crossterm::ExecutableCommand;
|
||||
use std::collections::BTreeMap;
|
||||
use std::io;
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::sync::mpsc::error::TryRecvError;
|
||||
use tokio::time::sleep;
|
||||
use tui::backend::CrosstermBackend;
|
||||
use tui::layout::{Constraint, Direction, Layout};
|
||||
use tui::style::{Color, Style};
|
||||
use tui::text::{Span, Spans};
|
||||
use tui::widgets::{BarChart, Block, Borders, Gauge, Paragraph};
|
||||
use tui::Terminal;
|
||||
use tokio::sync::mpsc::Receiver;
|
||||
|
||||
pub struct UI {
|
||||
pub n_run: usize,
|
||||
pub n_batch: usize,
|
||||
pub n_batch_done: usize,
|
||||
pub run_receiver: Receiver<()>,
|
||||
}
|
||||
|
||||
impl UI {
|
||||
pub async fn draw(
|
||||
mut self
|
||||
) -> Result<(), crossterm::ErrorKind> {
|
||||
crossterm::terminal::enable_raw_mode()?;
|
||||
io::stdout().execute(crossterm::terminal::EnterAlternateScreen)?;
|
||||
io::stdout().execute(crossterm::cursor::Hide)?;
|
||||
|
||||
let mut runs = Vec::new();
|
||||
|
||||
let mut terminal = {
|
||||
let backend = CrosstermBackend::new(io::stdout());
|
||||
Terminal::new(backend)?
|
||||
};
|
||||
|
||||
'outer: loop {
|
||||
let frame_start = Instant::now();
|
||||
loop {
|
||||
match self.run_receiver.try_recv() {
|
||||
Ok(run) => {
|
||||
// match report.as_ref() {
|
||||
// Ok(report) => *status_dist.entry(report.status).or_default() += 1,
|
||||
// Err(e) => *error_dist.entry(e.to_string()).or_default() += 1,
|
||||
// }
|
||||
// all.push(report);
|
||||
runs.push(run);
|
||||
}
|
||||
Err(TryRecvError::Empty) => {
|
||||
break;
|
||||
}
|
||||
Err(TryRecvError::Disconnected) => {
|
||||
// Application ends.
|
||||
break 'outer;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let draw_start = Instant::now();
|
||||
|
||||
let batch_progress = (self.n_batch_done as f64 / self.n_batch as f64).clamp(0.0, 1.0);
|
||||
let run_progress = (runs.len() as f64 / self.n_run as f64).clamp(0.0, 1.0);
|
||||
|
||||
terminal.draw(|f| {
|
||||
let row3 = Layout::default()
|
||||
.direction(Direction::Vertical)
|
||||
.constraints(
|
||||
[
|
||||
Constraint::Length(3),
|
||||
Constraint::Length(10),
|
||||
Constraint::Percentage(45),
|
||||
].as_ref(),
|
||||
).split(f.size());
|
||||
|
||||
let top = Layout::default()
|
||||
.direction(Direction::Horizontal)
|
||||
.constraints([
|
||||
Constraint::Percentage(50),
|
||||
Constraint::Percentage(50),
|
||||
].as_ref()).split(row3[0]);
|
||||
|
||||
let mid = Layout::default()
|
||||
.direction(Direction::Horizontal)
|
||||
.constraints([
|
||||
Constraint::Percentage(20),
|
||||
Constraint::Percentage(30),
|
||||
Constraint::Percentage(20),
|
||||
Constraint::Percentage(30),
|
||||
].as_ref()).split(row3[1]);
|
||||
|
||||
let prefill_text = Layout::default()
|
||||
.direction(Direction::Vertical)
|
||||
.constraints([
|
||||
Constraint::Length(5),
|
||||
Constraint::Length(5),
|
||||
].as_ref()).split(mid[0]);
|
||||
|
||||
let decode_text = Layout::default()
|
||||
.direction(Direction::Vertical)
|
||||
.constraints([
|
||||
Constraint::Length(5),
|
||||
Constraint::Length(5),
|
||||
].as_ref()).split(mid[2]);
|
||||
|
||||
let bottom = Layout::default()
|
||||
.direction(Direction::Horizontal)
|
||||
.constraints([
|
||||
Constraint::Percentage(25),
|
||||
Constraint::Percentage(25),
|
||||
Constraint::Percentage(25),
|
||||
Constraint::Percentage(25),
|
||||
].as_ref()).split(row3[2]);
|
||||
|
||||
let batch_gauge = Gauge::default()
|
||||
.block(Block::default().title("Total Progress").borders(Borders::ALL))
|
||||
.gauge_style(Style::default().fg(Color::White))
|
||||
.label(Span::raw(format!("{} / {}", self.n_batch_done, self.n_batch)))
|
||||
.ratio(batch_progress);
|
||||
f.render_widget(batch_gauge, top[0]);
|
||||
|
||||
let run_gauge = Gauge::default()
|
||||
.block(Block::default().title("Batch Progress").borders(Borders::ALL))
|
||||
.gauge_style(Style::default().fg(Color::White))
|
||||
.label(Span::raw(format!("{} / {}", runs.len(), self.n_run)))
|
||||
.ratio(run_progress);
|
||||
f.render_widget(run_gauge, top[1]);
|
||||
|
||||
let data = vec![0.0];
|
||||
|
||||
let prefill_latency_texts = statis_spans(&data, "ms", false);
|
||||
let prefill_throughput_texts = statis_spans(&data, "tokens/secs", false);
|
||||
|
||||
let prefill_latency_statics = Paragraph::new(prefill_latency_texts).block(
|
||||
Block::default()
|
||||
.title(Span::raw("Prefill Latency"))
|
||||
.borders(Borders::ALL),
|
||||
);
|
||||
f.render_widget(prefill_latency_statics, prefill_text[0]);
|
||||
|
||||
let prefill_throughput_statics = Paragraph::new(prefill_throughput_texts).block(
|
||||
Block::default()
|
||||
.title(Span::raw("Prefill Throughput"))
|
||||
.borders(Borders::ALL),
|
||||
);
|
||||
f.render_widget(prefill_throughput_statics, prefill_text[1]);
|
||||
})?;
|
||||
|
||||
let per_frame = Duration::from_secs(1) / 30 as u32;
|
||||
let elapsed = frame_start.elapsed();
|
||||
if per_frame > elapsed {
|
||||
sleep(per_frame - elapsed).await;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
io::stdout().execute(crossterm::terminal::LeaveAlternateScreen)?;
|
||||
crossterm::terminal::disable_raw_mode()?;
|
||||
io::stdout().execute(crossterm::cursor::Show)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn statis_spans<'a>(data: &Vec<f64>, unit: &'static str, color: bool) -> Vec<Spans<'a>> {
|
||||
vec![
|
||||
Spans::from(vec![Span::styled(
|
||||
format!(
|
||||
"Lowest: {:.4} {unit}",
|
||||
data
|
||||
.iter()
|
||||
.max_by(|a, b| a.total_cmp(b))
|
||||
.unwrap_or(&std::f64::NAN)
|
||||
),
|
||||
Style::default().fg(Color::Reset),
|
||||
)]),
|
||||
Spans::from(vec![Span::styled(
|
||||
format!(
|
||||
"Highest: {:.4} {unit}",
|
||||
data
|
||||
.iter()
|
||||
.min_by(|a, b| a.total_cmp(b))
|
||||
.unwrap_or(&std::f64::NAN)
|
||||
),
|
||||
Style::default().fg(Color::Reset),
|
||||
)]),
|
||||
Spans::from(vec![Span::styled(
|
||||
format!(
|
||||
"Average: {:.4} {unit}",
|
||||
data
|
||||
.iter()
|
||||
.sum::<f64>()
|
||||
/ data.len() as f64
|
||||
),
|
||||
Style::default().fg(Color::Reset),
|
||||
)]),
|
||||
]
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user