From 295831a481241d3d06b49f646a40f27b1297fab5 Mon Sep 17 00:00:00 2001 From: Olivier Dehaene Date: Sat, 8 Oct 2022 12:30:12 +0200 Subject: [PATCH] Init --- README.md | 37 + proto/generate.proto | 83 + router/.gitignore | 1 + router/Cargo.lock | 2373 +++++++++++++++++ router/Cargo.toml | 28 + router/client/Cargo.toml | 19 + router/client/build.rs | 14 + router/client/src/client.rs | 104 + router/client/src/lib.rs | 32 + router/client/src/pb/.gitignore | 1 + router/client/src/sharded_client.rs | 106 + router/src/db.rs | 129 + router/src/infer.rs | 130 + router/src/main.rs | 125 + server/.DS_Store | Bin 0 -> 6148 bytes server/Makefile | 20 + server/README.md | 15 + server/bloom_inference/.DS_Store | Bin 0 -> 6148 bytes server/bloom_inference/__init__.py | 0 .../__pycache__/__init__.cpython-39.pyc | Bin 0 -> 158 bytes .../__pycache__/cache.cpython-39.pyc | Bin 0 -> 2296 bytes .../__pycache__/model.cpython-39.pyc | Bin 0 -> 9759 bytes .../__pycache__/server.cpython-39.pyc | Bin 0 -> 3683 bytes .../__pycache__/shard_model.cpython-39.pyc | Bin 0 -> 3058 bytes .../__pycache__/utils.cpython-39.pyc | Bin 0 -> 3184 bytes server/bloom_inference/cache.py | 48 + server/bloom_inference/main.py | 30 + server/bloom_inference/model.py | 428 +++ server/bloom_inference/pb/__init__.py | 0 server/bloom_inference/pb/__init__.py-e | 0 .../pb/__pycache__/__init__.cpython-39.pyc | Bin 0 -> 161 bytes .../__pycache__/generate_pb2.cpython-39.pyc | Bin 0 -> 2182 bytes .../generate_pb2_grpc.cpython-39.pyc | Bin 0 -> 4355 bytes server/bloom_inference/pb/generate_pb2.py | 43 + server/bloom_inference/pb/generate_pb2.py-e | 43 + .../bloom_inference/pb/generate_pb2_grpc.py | 169 ++ .../bloom_inference/pb/generate_pb2_grpc.py-e | 169 ++ server/bloom_inference/prepare_weights.py | 124 + server/bloom_inference/server.py | 91 + server/bloom_inference/shard_model.py | 102 + server/bloom_inference/utils.py | 95 + server/poetry.lock | 480 ++++ server/pyproject.toml | 21 + 43 files changed, 5060 insertions(+) create mode 100644 README.md create mode 100644 proto/generate.proto create mode 100644 router/.gitignore create mode 100644 router/Cargo.lock create mode 100644 router/Cargo.toml create mode 100644 router/client/Cargo.toml create mode 100644 router/client/build.rs create mode 100644 router/client/src/client.rs create mode 100644 router/client/src/lib.rs create mode 100644 router/client/src/pb/.gitignore create mode 100644 router/client/src/sharded_client.rs create mode 100644 router/src/db.rs create mode 100644 router/src/infer.rs create mode 100644 router/src/main.rs create mode 100644 server/.DS_Store create mode 100644 server/Makefile create mode 100644 server/README.md create mode 100644 server/bloom_inference/.DS_Store create mode 100644 server/bloom_inference/__init__.py create mode 100644 server/bloom_inference/__pycache__/__init__.cpython-39.pyc create mode 100644 server/bloom_inference/__pycache__/cache.cpython-39.pyc create mode 100644 server/bloom_inference/__pycache__/model.cpython-39.pyc create mode 100644 server/bloom_inference/__pycache__/server.cpython-39.pyc create mode 100644 server/bloom_inference/__pycache__/shard_model.cpython-39.pyc create mode 100644 server/bloom_inference/__pycache__/utils.cpython-39.pyc create mode 100644 server/bloom_inference/cache.py create mode 100644 server/bloom_inference/main.py create mode 100644 server/bloom_inference/model.py create mode 100644 server/bloom_inference/pb/__init__.py create mode 100644 server/bloom_inference/pb/__init__.py-e create mode 100644 server/bloom_inference/pb/__pycache__/__init__.cpython-39.pyc create mode 100644 server/bloom_inference/pb/__pycache__/generate_pb2.cpython-39.pyc create mode 100644 server/bloom_inference/pb/__pycache__/generate_pb2_grpc.cpython-39.pyc create mode 100644 server/bloom_inference/pb/generate_pb2.py create mode 100644 server/bloom_inference/pb/generate_pb2.py-e create mode 100644 server/bloom_inference/pb/generate_pb2_grpc.py create mode 100644 server/bloom_inference/pb/generate_pb2_grpc.py-e create mode 100644 server/bloom_inference/prepare_weights.py create mode 100644 server/bloom_inference/server.py create mode 100644 server/bloom_inference/shard_model.py create mode 100644 server/bloom_inference/utils.py create mode 100644 server/poetry.lock create mode 100644 server/pyproject.toml diff --git a/README.md b/README.md new file mode 100644 index 000000000..fe7bb4439 --- /dev/null +++ b/README.md @@ -0,0 +1,37 @@ +# BLOOM Inference + +A Rust and gRPC server for BLOOM Inference. + +## Install + +```shell +cd server +pip install . +``` + +``` +cd router +cargo build --release +``` + +## Run + +```shell +python server/bloom_inference/main.py bigscience/bloom --num-gpus 8 --shard-directory /dev/shm/models +``` + +```shell +./router/target/release/router +``` + +## TODO: + +- [ ] Improve model download + - Store "shardable" layers separately and layer by layer +- [ ] Add batching args to router CLI +- [ ] Add docstrings + comments everywhere as the codebase is fairly complicated +- [ ] Add tests +- [ ] Add shutdown logic in router and server +- [ ] Improve multi-processing logic in server +- [ ] Improve error handling everywhere +- [ ] Improve past key layer indexing? \ No newline at end of file diff --git a/proto/generate.proto b/proto/generate.proto new file mode 100644 index 000000000..324b9206e --- /dev/null +++ b/proto/generate.proto @@ -0,0 +1,83 @@ +syntax = "proto3"; + +package generate.v1; + +service TextGeneration { + /// Service discovery + rpc ServiceDiscovery(Empty) returns (ServiceDiscoveryResponse) {} + /// Empties batch cache + rpc ClearCache(Empty) returns (Empty); + /// Generate tokens for a batch without cache + rpc Generate(Batch) returns (Response); + /// Generate tokens for a batch with cache + rpc GenerateWithCache(BatchCached) returns (Response); +} + +message ServiceDiscoveryResponse { + repeated string urls = 1; +} + +message LogitsWarperParameters { + float temperature = 1; + uint32 top_k = 2; + float top_p = 3; + bool do_sample = 4; +} + +message Request { + /// Request ID + uint64 id = 1; + /// The generation context + string inputs = 2; + /// Logits Warper Parameters + LogitsWarperParameters parameters = 3; + /// Stopping criteria + uint32 max_new_tokens = 4; +} + +message Batch { + /// Batch ID + uint64 id = 1; + /// Individual requests + repeated Request requests = 2; +} + +message BatchCached { + /// Batch ID + uint64 id = 1; + /// Request ids within cache + repeated uint64 request_ids = 2; + /// Cache IDs + repeated uint64 batch_cached_ids = 3; + /// Batch size (sum of all batch sizes) + uint32 total_batch_size = 4; + /// Max sequence length + uint32 max_sequence_length = 5; +} + +message FinishedGeneration { + /// ID of the original request + uint64 id = 1; + /// Output + string output = 2; +} + +message CacheEntry { + /// Cache ID; same as batch ID + uint64 id = 1; + /// Requests present in cache entry + repeated uint64 request_ids = 2; + /// Sequence length + uint32 sequence_length = 3; +} + +message Response { + /// Finished requests (optional) + repeated FinishedGeneration finished = 1; + /// Cache entry (optional) + optional CacheEntry cache_entry = 2; +} + + +// Represent an empty message. +message Empty {} \ No newline at end of file diff --git a/router/.gitignore b/router/.gitignore new file mode 100644 index 000000000..ea8c4bf7f --- /dev/null +++ b/router/.gitignore @@ -0,0 +1 @@ +/target diff --git a/router/Cargo.lock b/router/Cargo.lock new file mode 100644 index 000000000..797613457 --- /dev/null +++ b/router/Cargo.lock @@ -0,0 +1,2373 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "aho-corasick" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4f55bd91a0978cbfd91c457a164bab8b4001c833b7f323132c0a4e1922dd44e" +dependencies = [ + "memchr", +] + +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + +[[package]] +name = "anyhow" +version = "1.0.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98161a4e3e2184da77bb14f02184cdd111e83bbbcc9979dfee3c44b9a85f5602" + +[[package]] +name = "async-stream" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dad5c83079eae9969be7fadefe640a1c566901f05ff91ab221de4b6f68d9507e" +dependencies = [ + "async-stream-impl", + "futures-core", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "async-trait" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76464446b8bc32758d7e88ee1a804d9914cd9b1cb264c029899680b0be29826f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "block-buffer" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bloom-inference" +version = "0.1.0" +dependencies = [ + "bloom-inference-client", + "futures", + "parking_lot", + "poem", + "serde", + "serde_json", + "tokenizers", + "tokio", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "bloom-inference-client" +version = "0.1.0" +dependencies = [ + "futures", + "prost", + "thiserror", + "tokio", + "tonic", + "tonic-build", + "tower", + "tracing", + "tracing-error", +] + +[[package]] +name = "bumpalo" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1ad822118d20d2c234f427000d5acc36eabe1e29a348c89b63dd60b13f28e5d" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "bytes" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db" + +[[package]] +name = "bzip2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6afcd980b5f3a45017c57e57a2fcccbb351cc43a356ce117ef760ef8052b89b0" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "cached-path" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f1c56d30236522ab3393a08746b138d4e16372001f42d29c88d513aeb8ab7ef" +dependencies = [ + "flate2", + "fs2", + "glob", + "indicatif 0.16.2", + "log", + "rand", + "reqwest", + "serde", + "serde_json", + "sha2", + "tar", + "tempfile", + "thiserror", + "zip", + "zip-extensions", +] + +[[package]] +name = "cc" +version = "1.0.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "clap" +version = "2.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +dependencies = [ + "ansi_term", + "atty", + "bitflags", + "strsim 0.8.0", + "textwrap", + "unicode-width", + "vec_map", +] + +[[package]] +name = "console" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c050367d967ced717c04b65d8c619d863ef9292ce0c5760028655a2fb298718c" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "terminal_size", + "unicode-width", + "winapi", +] + +[[package]] +name = "core-foundation" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" + +[[package]] +name = "cpufeatures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f916dfc5d356b0ed9dae65f1db9fc9770aa2851d2662b988ccf4fe3516e86348" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edbafec5fa1f196ca66527c1b12c2ec4745ca14b50f1ad8f9f6f720b55d11fac" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "darling" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d706e75d87e35569db781a9b5e2416cff1236a47ed380831f959382ccd5f858" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c960ae2da4de88a91b2d920c2a7233b400bc33cb28453a2987822d8392519b" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim 0.9.3", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b5a2f4ac4969822c62224815d069952656cadc7084fdca9751e6d959189b72" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "derive_builder" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2658621297f2cf68762a6f7dc0bb7e1ff2cfd6583daef8ee0fed6f7ec468ec0" +dependencies = [ + "darling", + "derive_builder_core", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "derive_builder_core" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2791ea3e372c8495c0bc2033991d76b512cd799d07491fbd6890124db9458bef" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "digest" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adfbc57365a37acbd2ebf2b64d7e69bb766e2fea813521ed536f5d0520dcf86c" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "dirs" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30baa043103c9d0c2a57cf537cc2f35623889dc0d405e6c3cccfadbc81c71309" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + +[[package]] +name = "either" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" + +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + +[[package]] +name = "encoding_rs" +version = "0.8.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "esaxx-rs" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f748b253ceca9fed5f42f8b5ceb3851e93102199bc25b64b65369f76e5c0a35" +dependencies = [ + "cc", +] + +[[package]] +name = "fastrand" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a407cfaa3385c4ae6b23e84623d48c2798d06e3e6a1878f7f59f17b3f86499" +dependencies = [ + "instant", +] + +[[package]] +name = "filetime" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e94a7bbaa59354bc20dd75b67f23e2797b4490e9d6928203fb105c79e448c86c" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "windows-sys", +] + +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + +[[package]] +name = "flate2" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "futures" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f21eda599937fba36daeb58a22e8f5cee2d14c4a17b5b7739c7c8e5e3b8230c" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30bdd20c28fadd505d0fd6712cdfcb0d4b5648baf45faef7f852afb2399bb050" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e5aa3de05362c3fb88de6531e6296e85cde7739cccad4b9dfeeb7f6ebce56bf" + +[[package]] +name = "futures-executor" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ff63c23854bee61b6e9cd331d523909f238fc7636290b96826e9cfa5faa00ab" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbf4d2a7a308fd4578637c0b17c7e1c7ba127b8f6ba00b29f717e9655d85eb68" + +[[package]] +name = "futures-macro" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42cd15d1c7456c04dbdf7e88bcd69760d74f3a798d6444e16974b505b0e62f17" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b20ba5a92e727ba30e72834706623d94ac93a725410b6a6b6fbc1b07f7ba56" + +[[package]] +name = "futures-task" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6508c467c73851293f390476d4491cf4d227dbabcd4170f3bb6044959b294f1" + +[[package]] +name = "futures-util" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44fb6cb1be61cc1d2e43b262516aafcf63b241cffdb1d3fa115f91d9c7b09c90" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", +] + +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" + +[[package]] +name = "h2" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca32592cf21ac7ccab1825cd87f6c9b3d9022c44d086172ed0966bec8af30be" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util 0.7.4", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "headers" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3e372db8e5c0d213e0cd0b9be18be2aca3d44cf2fe30a9d46a65581cd454584" +dependencies = [ + "base64", + "bitflags", + "bytes", + "headers-core", + "http", + "httpdate", + "mime", + "sha1", +] + +[[package]] +name = "headers-core" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7f66481bfee273957b1f20485a4ff3362987f85b2c236580d81b4eb7a326429" +dependencies = [ + "http", +] + +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "http" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" + +[[package]] +name = "hyper" +version = "0.14.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02c929dc5c39e335a03c405292728118860721b10190d98c2a0f0efd5baafbac" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] +name = "indicatif" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7baab56125e25686df467fe470785512329883aab42696d661247aca2a2896e4" +dependencies = [ + "console", + "lazy_static", + "number_prefix 0.3.0", + "regex", +] + +[[package]] +name = "indicatif" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d207dc617c7a380ab07ff572a6e52fa202a2a8f355860ac9c38e23f8196be1b" +dependencies = [ + "console", + "lazy_static", + "number_prefix 0.4.0", + "regex", +] + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "ipnet" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b" + +[[package]] +name = "itertools" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f56a2d0bc861f9165be4eb3442afd3c236d8a98afd426f65d92324ae1091a484" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8af84674fe1f223a982c933a0ee1086ac4d4052aa0fb8060c12c6ad838e754" + +[[package]] +name = "js-sys" +version = "0.3.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.134" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "329c933548736bc49fd575ee68c89e8be4d260064184389a5b77517cddd99ffb" + +[[package]] +name = "lock_api" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "macro_rules_attribute" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "258c86475e1616d6f2d8f5227cfaabd3dae1f6d5388b9597df8a199d4497aba7" +dependencies = [ + "macro_rules_attribute-proc_macro", + "paste", +] + +[[package]] +name = "macro_rules_attribute-proc_macro" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26a8d2502d5aa4d411ef494ba7470eb299f05725179ce3b5de77aa01a9ffdea" + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "memoffset" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" +dependencies = [ + "autocfg", +] + +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96590ba8f175222643a85693f33d26e9c8a015f599c216509b1a6894af675d34" +dependencies = [ + "adler", +] + +[[package]] +name = "mio" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57ee1c23c7c63b0c9250c339ffdc69255f110b298b901b9f6c82547b7b87caaf" +dependencies = [ + "libc", + "log", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys", +] + +[[package]] +name = "multimap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" + +[[package]] +name = "native-tls" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd7e2f3618557f980e0b17e8856252eee3c97fa12c54dff0ca290fb6266ca4a9" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "nom" +version = "7.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "num_cpus" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "number_prefix" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a" + +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + +[[package]] +name = "once_cell" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e82dad04139b71a90c080c8463fe0dc7902db5192d939bd0950f074d014339e1" + +[[package]] +name = "onig" +version = "6.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c4b31c8722ad9171c6d77d3557db078cab2bd50afcc9d09c8b315c59df8ca4f" +dependencies = [ + "bitflags", + "libc", + "once_cell", + "onig_sys", +] + +[[package]] +name = "onig_sys" +version = "69.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b829e3d7e9cc74c7e315ee8edb185bf4190da5acde74afd7fc59c35b1f086e7" +dependencies = [ + "cc", + "pkg-config", +] + +[[package]] +name = "openssl" +version = "0.10.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12fc0523e3bd51a692c8850d075d74dc062ccf251c0110668cbd921917118a13" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5230151e44c0f05157effb743e8d517472843121cf9243e8b81393edb5acd9ce" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-sys", +] + +[[package]] +name = "paste" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1de2e551fb905ac83f73f7aedf2f0cb4a0da7e35efa24a202a936269f1f18e1" + +[[package]] +name = "percent-encoding" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" + +[[package]] +name = "petgraph" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5014253a1331579ce62aa67443b4a658c5e7dd03d4bc6d302b94474888143" +dependencies = [ + "fixedbitset", + "indexmap", +] + +[[package]] +name = "pin-project" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" + +[[package]] +name = "poem" +version = "1.3.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2992ba72908e36200671c0f3a692992ced894b3b2bbe2b2dc6dfbffea6e2c85a" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "headers", + "http", + "hyper", + "mime", + "parking_lot", + "percent-encoding", + "pin-project-lite", + "poem-derive", + "regex", + "rfc7239", + "serde", + "serde_json", + "serde_urlencoded", + "smallvec", + "thiserror", + "tokio", + "tokio-stream", + "tokio-util 0.7.4", + "tracing", +] + +[[package]] +name = "poem-derive" +version = "1.3.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f535d4331a22610b98ca48f98bae9bda0c654da89b9ae10a1830fa9edfd8f36" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" + +[[package]] +name = "proc-macro-crate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eda0fc3b0fb7c975631757e14d9049da17374063edb6ebbcbc54d880d4fe94e9" +dependencies = [ + "once_cell", + "thiserror", + "toml", +] + +[[package]] +name = "proc-macro2" +version = "1.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94e2ef8dbfc347b10c094890f778ee2e36ca9bb4262e86dc99cd217e35f3470b" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "444879275cb4fd84958b1a1d5420d15e6fcf7c235fe47f053c9c2a80aceb6001" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62941722fb675d463659e49c4f3fe1fe792ff24fe5bbaa9c08cd3b98a1c354f5" +dependencies = [ + "bytes", + "heck", + "itertools 0.10.5", + "lazy_static", + "log", + "multimap", + "petgraph", + "prost", + "prost-types", + "regex", + "tempfile", + "which", +] + +[[package]] +name = "prost-derive" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9cc1a3263e07e0bf68e96268f37665207b49560d98739662cdfaae215c720fe" +dependencies = [ + "anyhow", + "itertools 0.10.5", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "prost-types" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "534b7a0e836e3c482d2693070f982e39e7611da9695d4d1f5a4b186b51faef0a" +dependencies = [ + "bytes", + "prost", +] + +[[package]] +name = "quote" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rayon" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd99e5772ead8baa5215278c9b15bf92087709e9c1b2d1f97cdb5a183c933a7d" +dependencies = [ + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-cond" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd1259362c9065e5ea39a789ef40b1e3fd934c94beb7b5ab3ac6629d3b5e7cb7" +dependencies = [ + "either", + "itertools 0.8.2", + "rayon", +] + +[[package]] +name = "rayon-core" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "258bcdb5ac6dad48491bb2992db6b7cf74878b0384908af124823d118c99683f" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "num_cpus", +] + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags", +] + +[[package]] +name = "redox_users" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +dependencies = [ + "getrandom", + "redox_syscall", + "thiserror", +] + +[[package]] +name = "regex" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" + +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + +[[package]] +name = "reqwest" +version = "0.11.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "431949c384f4e2ae07605ccaa56d1d9d2ecdb5cadd4f9577ccfab29f2e5149fc" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + +[[package]] +name = "rfc7239" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "087317b3cf7eb481f13bd9025d729324b7cd068d6f470e2d76d049e191f5ba47" +dependencies = [ + "uncased", +] + +[[package]] +name = "ryu" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" + +[[package]] +name = "schannel" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88d6731146462ea25d9244b2ed5fd1d716d25c52e4d54aa4fb0f3c4e9854dbe2" +dependencies = [ + "lazy_static", + "windows-sys", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "security-framework" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bc1bb97804af6631813c55739f771071e0f2ed33ee20b68c86ec505d906356c" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0160a13a177a45bfb43ce71c01580998474f556ad854dcbca936dd2841a5c556" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.145" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728eb6351430bccb993660dfffc5a72f91ccc1295abaa8ce19b27ebe4f75568b" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.145" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81fa1584d3d1bcacd84c277a0dfe21f5b0f6accf4a23d04d4c6d61f1af522b4c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.85" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e55a28e3aaef9d5ce0506d0a14dbba8054ddc7e499ef522dd8b26859ec9d4a44" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha1" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "slab" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" + +[[package]] +name = "socket2" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "spm_precompiled" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" +dependencies = [ + "base64", + "nom", + "serde", + "unicode-segmentation", +] + +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + +[[package]] +name = "strsim" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6446ced80d6c486436db5c078dde11a9f73d42b57fb273121e160b84f63d894c" + +[[package]] +name = "syn" +version = "1.0.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e90cde112c4b9690b8cbe810cba9ddd8bc1d7472e2cae317b69e9438c1cba7d2" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tar" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b55807c0344e1e6c04d7c965f5289c39a8d94ae23ed5c0b57aabac549f871c6" +dependencies = [ + "filetime", + "libc", + "xattr", +] + +[[package]] +name = "tempfile" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" +dependencies = [ + "cfg-if", + "fastrand", + "libc", + "redox_syscall", + "remove_dir_all", + "winapi", +] + +[[package]] +name = "terminal_size" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "thiserror" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180" +dependencies = [ + "once_cell", +] + +[[package]] +name = "time" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +dependencies = [ + "libc", + "wasi 0.10.0+wasi-snapshot-preview1", + "winapi", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + +[[package]] +name = "tokenizers" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b08ede6742d7a59d58c71da8a6fa21bedc433dca2e855e439274d08df1170" +dependencies = [ + "aho-corasick", + "cached-path", + "clap", + "derive_builder", + "dirs", + "esaxx-rs", + "getrandom", + "indicatif 0.15.0", + "itertools 0.9.0", + "lazy_static", + "log", + "macro_rules_attribute", + "onig", + "paste", + "rand", + "rayon", + "rayon-cond", + "regex", + "regex-syntax", + "reqwest", + "serde", + "serde_json", + "spm_precompiled", + "thiserror", + "unicode-normalization-alignments", + "unicode-segmentation", + "unicode_categories", +] + +[[package]] +name = "tokio" +version = "1.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9e03c497dc955702ba729190dc4aac6f2a0ce97f913e5b1b5912fc5039d9099" +dependencies = [ + "autocfg", + "bytes", + "libc", + "memchr", + "mio", + "num_cpus", + "parking_lot", + "pin-project-lite", + "socket2", + "tokio-macros", + "winapi", +] + +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-macros" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9724f9a975fb987ef7a3cd9be0350edcbe130698af5b8f7a631e23d42d052484" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6edf2d6bc038a43d31353570e27270603f4648d18f5ed10c0e179abe43255af" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.6.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36943ee01a6d67977dd3f84a5a1d2efeb4ada3a1ae771cadfaa535d9d9fc6507" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "log", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "toml" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" +dependencies = [ + "serde", +] + +[[package]] +name = "tonic" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff08f4649d10a70ffa3522ca559031285d8e421d727ac85c60825761818f5d0a" +dependencies = [ + "async-stream", + "async-trait", + "base64", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost", + "prost-derive", + "tokio", + "tokio-stream", + "tokio-util 0.6.10", + "tower", + "tower-layer", + "tower-service", + "tracing", + "tracing-futures", +] + +[[package]] +name = "tonic-build" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9403f1bafde247186684b230dc6f38b5cd514584e8bec1dd32514be4745fa757" +dependencies = [ + "proc-macro2", + "prost-build", + "quote", + "syn", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap", + "pin-project", + "pin-project-lite", + "rand", + "slab", + "tokio", + "tokio-util 0.7.4", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "343bc9466d3fe6b0f960ef45960509f84480bf4fd96f92901afe7ff3df9d3a62" + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tracing" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fce9567bd60a67d08a16488756721ba392f24f29006402881e43b19aac64307" +dependencies = [ + "cfg-if", + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11c75893af559bc8e10716548bdef5cb2b983f8e637db9d0e15126b61b484ee2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeea4303076558a00714b823f9ad67d58a3bbda1df83d8827d21193156e22f7" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-error" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d686ec1c0f384b1277f097b2f279a2ecc11afe8c133c1aabf036a27cb4cd206e" +dependencies = [ + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "tracing-futures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" +dependencies = [ + "pin-project", + "tracing", +] + +[[package]] +name = "tracing-log" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" +dependencies = [ + "lazy_static", + "log", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60db860322da191b40952ad9affe65ea23e7dd6a5c442c2c42865810c6ab8e6b" +dependencies = [ + "ansi_term", + "sharded-slab", + "smallvec", + "thread_local", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + +[[package]] +name = "typenum" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" + +[[package]] +name = "uncased" +version = "0.9.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09b01702b0fd0b3fadcf98e098780badda8742d4f4a7676615cad90e8ac73622" +dependencies = [ + "version_check", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" + +[[package]] +name = "unicode-ident" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcc811dc4066ac62f84f11307873c4850cb653bfa9b1719cee2bd2204a4bc5dd" + +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-normalization-alignments" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" +dependencies = [ + "smallvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fdbf052a0783de01e944a6ce7a8cb939e295b1e7be835a1112c3b9a7f047a5a" + +[[package]] +name = "unicode-width" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" + +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + +[[package]] +name = "url" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23639446165ca5a5de86ae1d8896b737ae80319560fbaa4c2887b7da6e7ebd7d" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" + +[[package]] +name = "web-sys" +version = "0.3.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "which" +version = "4.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c831fbbee9e129a8cf93e7747a82da9d95ba8e16621cae60ec2cdc849bacb7b" +dependencies = [ + "either", + "libc", + "once_cell", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" +dependencies = [ + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" + +[[package]] +name = "windows_i686_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" + +[[package]] +name = "windows_i686_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" + +[[package]] +name = "winreg" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +dependencies = [ + "winapi", +] + +[[package]] +name = "xattr" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d1526bbe5aaeb5eb06885f4d987bcdfa5e23187055de9b83fe00156a821fabc" +dependencies = [ + "libc", +] + +[[package]] +name = "zip" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93ab48844d61251bb3835145c521d88aa4031d7139e8485990f60ca911fa0815" +dependencies = [ + "byteorder", + "bzip2", + "crc32fast", + "flate2", + "thiserror", + "time", +] + +[[package]] +name = "zip-extensions" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a64c3c977bc3434ce2d4bcea8ad3c644672de0f2c402b72b9171ca80a8885d14" +dependencies = [ + "zip", +] diff --git a/router/Cargo.toml b/router/Cargo.toml new file mode 100644 index 000000000..901a14f24 --- /dev/null +++ b/router/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "bloom-inference" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +bloom-inference-client = { path = "client" } +futures = "0.3.24" +parking_lot = "0.12.1" +poem = "1.3.45" +serde = "1.0.145" +serde_json = "1.0.85" +tokenizers = "0.13.0" +tokio = { version = "1.21.1", features = ["rt-multi-thread", "parking_lot", "sync"] } +tracing = "0.1.36" +tracing-subscriber = "0.3.15" + +[workspace] +members = [ + "client", +] + +[profile.release] +debug = 1 +incremental = true +lto = "off" diff --git a/router/client/Cargo.toml b/router/client/Cargo.toml new file mode 100644 index 000000000..e4f2fa0f8 --- /dev/null +++ b/router/client/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "bloom-inference-client" +version = "0.1.0" +edition = "2021" + +[dependencies] +futures = "0.3.24" +#grpc-error-details = { path = "../../grpc-error-details" } +#grpc-metadata = { path = "../../grpc-metadata" } +prost = "^0.9" +thiserror = "1.0.37" +tokio = { version = "1.21.2", features = ["sync"] } +tonic = "^0.6" +tower = "^0.4" +tracing = "^0.1" +tracing-error = "^0.2" + +[build-dependencies] +tonic-build = "0.6.2" diff --git a/router/client/build.rs b/router/client/build.rs new file mode 100644 index 000000000..1876b2174 --- /dev/null +++ b/router/client/build.rs @@ -0,0 +1,14 @@ +use std::fs; + +fn main() -> Result<(), Box> { + fs::create_dir("src/pb").unwrap_or(()); + tonic_build::configure() + .build_client(true) + .build_server(false) + .out_dir("src/pb") + .include_file("mod.rs") + .compile(&["../../proto/generate.proto"], &["../../proto"]) + .unwrap_or_else(|e| panic!("protobuf compilation failed: {}", e)); + + Ok(()) +} diff --git a/router/client/src/client.rs b/router/client/src/client.rs new file mode 100644 index 000000000..7f68e2525 --- /dev/null +++ b/router/client/src/client.rs @@ -0,0 +1,104 @@ +use crate::pb::generate::v1::text_generation_client::TextGenerationClient; +use crate::pb::generate::v1::*; +use crate::Result; +use std::time::Duration; +use tonic::transport::{Channel, Uri}; +use tower::timeout::Timeout; +use tracing::*; + +/// BLOOM Inference gRPC client +#[derive(Clone)] +pub struct Client { + stub: TextGenerationClient>, +} + +impl Client { + /// Returns a client connected to the given url. Requests exceeding timeout will fail. + pub async fn connect(uri: Uri, timeout: Duration) -> Self { + let channel = Channel::builder(uri) + .connect() + .await + .expect("Transport error"); + let timeout_channel = Timeout::new(channel, timeout); + + Self { + stub: TextGenerationClient::new(timeout_channel), + } + } + + /// Returns a client connected to the given unix socket. Requests exceeding timeout will fail. + pub async fn connect_uds(path: String, timeout: Duration) -> Self { + let channel = Channel::from_shared(format!("http://[::]:50051")) + .unwrap() + .connect_with_connector(tower::service_fn(move |_: Uri| { + tokio::net::UnixStream::connect(path.clone()) + })) + .await + .expect("Transport error"); + let timeout_channel = Timeout::new(channel, timeout); + + Self { + stub: TextGenerationClient::new(timeout_channel), + } + } + + #[instrument(skip(self))] + pub async fn service_discovery(&mut self) -> Result> { + let request = tonic::Request::new(Empty {}); + let response = self + .stub + .service_discovery(request) + .instrument(info_span!("service_discovery")) + .await?; + let urls = response + .into_inner() + .urls + .into_iter() + .map(|url| match url.strip_prefix("unix://") { + None => url, + Some(stripped_url) => stripped_url.to_string(), + }) + .collect(); + Ok(urls) + } + + #[instrument(skip(self))] + pub async fn clear_cache(&mut self) -> Result<()> { + let request = tonic::Request::new(Empty {}); + self.stub + .clear_cache(request) + .instrument(info_span!("clear_cache")) + .await?; + Ok(()) + } + + #[instrument(skip(self))] + pub async fn generate( + &mut self, + request: Batch, + ) -> Result<(Vec, Option)> { + let request = tonic::Request::new(request); + let response = self + .stub + .generate(request) + .instrument(info_span!("generate")) + .await? + .into_inner(); + Ok((response.finished, response.cache_entry)) + } + + #[instrument(skip(self))] + pub async fn generate_with_cache( + &mut self, + request: BatchCached, + ) -> Result<(Vec, Option)> { + let request = tonic::Request::new(request); + let response = self + .stub + .generate_with_cache(request) + .instrument(info_span!("generate_with_cache")) + .await? + .into_inner(); + Ok((response.finished, response.cache_entry)) + } +} diff --git a/router/client/src/lib.rs b/router/client/src/lib.rs new file mode 100644 index 000000000..aca9f65bc --- /dev/null +++ b/router/client/src/lib.rs @@ -0,0 +1,32 @@ +//! BLOOM Inference gRPC client library + +mod client; +mod pb; +mod sharded_client; + +pub use client::Client; +pub use pb::generate::v1::{ + Batch, BatchCached, CacheEntry, FinishedGeneration, LogitsWarperParameters, Request, +}; +pub use sharded_client::ShardedClient; +use thiserror::Error; +pub use tonic::transport::Uri; +use tonic::Status; + +#[derive(Error, Debug, Clone)] +#[error("Text generation client error: {msg:?}")] +pub struct ClientError { + msg: String, + // source: Status, +} + +impl From for ClientError { + fn from(err: Status) -> Self { + Self { + msg: err.to_string(), + // source: err, + } + } +} + +pub type Result = std::result::Result; diff --git a/router/client/src/pb/.gitignore b/router/client/src/pb/.gitignore new file mode 100644 index 000000000..b46a4c427 --- /dev/null +++ b/router/client/src/pb/.gitignore @@ -0,0 +1 @@ +*.rs \ No newline at end of file diff --git a/router/client/src/sharded_client.rs b/router/client/src/sharded_client.rs new file mode 100644 index 000000000..6856a9bcb --- /dev/null +++ b/router/client/src/sharded_client.rs @@ -0,0 +1,106 @@ +use crate::Result; +use crate::{Batch, BatchCached, CacheEntry, Client, FinishedGeneration}; +use futures::future::join_all; +use std::time::Duration; +use tokio::sync::{broadcast, mpsc}; +use tonic::transport::Uri; + +#[derive(Clone, Debug)] +enum Command { + Generate( + Batch, + mpsc::Sender, Option)>>, + ), + GenerateWithCache( + BatchCached, + mpsc::Sender, Option)>>, + ), + ClearCache(mpsc::Sender>), +} + +async fn client_task(mut client: Client, mut request_subscriber: broadcast::Receiver) { + while let Ok(message) = request_subscriber.recv().await { + match message { + Command::Generate(batch, response_tx) => { + let result = client.generate(batch).await; + response_tx.try_send(result).unwrap_or(()); + } + Command::GenerateWithCache(batch_cached, response_tx) => { + let result = client.generate_with_cache(batch_cached).await; + response_tx.try_send(result).unwrap_or(()); + } + Command::ClearCache(response_tx) => { + let result = client.clear_cache().await; + response_tx.try_send(result).unwrap_or(()); + } + }; + } +} + +pub struct ShardedClient { + request_tx: broadcast::Sender, +} + +impl ShardedClient { + fn new(mut clients: Vec) -> Self { + let (request_tx, _) = broadcast::channel(1); + + for client in clients.drain(..) { + let request_subscriber = request_tx.subscribe(); + tokio::spawn(client_task(client, request_subscriber)); + } + + Self { request_tx } + } + + async fn from_master_client(mut master_client: Client) -> Self { + let uris = master_client.service_discovery().await.unwrap(); + let futures = uris + .into_iter() + .map(|path| Client::connect_uds(path, Duration::from_secs(5))); + let clients = join_all(futures).await; + Self::new(clients) + } + + /// Returns a client connected to the given url. Requests exceeding timeout will fail. + pub async fn connect(uri: Uri, timeout: Duration) -> Self { + let master_client = Client::connect(uri, timeout).await; + Self::from_master_client(master_client).await + } + + /// Returns a client connected to the given unix socket. Requests exceeding timeout will fail. + pub async fn connect_uds(path: String, timeout: Duration) -> Self { + let master_client = Client::connect_uds(path, timeout).await; + Self::from_master_client(master_client).await + } + + pub async fn generate( + &self, + batch: Batch, + ) -> Result<(Vec, Option)> { + let (response_tx, mut response_rx) = mpsc::channel(1); + self.request_tx + .send(Command::Generate(batch, response_tx)) + .unwrap(); + response_rx.recv().await.unwrap() + } + + pub async fn generate_with_cache( + &self, + batch_cached: BatchCached, + ) -> Result<(Vec, Option)> { + let (response_tx, mut response_rx) = mpsc::channel(1); + self.request_tx + .send(Command::GenerateWithCache(batch_cached, response_tx)) + .unwrap(); + response_rx.recv().await.unwrap() + } + + pub async fn clear_cache(&self) -> Result<()> { + let (response_tx, mut response_rx) = mpsc::channel(1); + self.request_tx + .send(Command::ClearCache(response_tx)) + .unwrap(); + response_rx.recv().await.unwrap() + } +} diff --git a/router/src/db.rs b/router/src/db.rs new file mode 100644 index 000000000..3dd98d94f --- /dev/null +++ b/router/src/db.rs @@ -0,0 +1,129 @@ +/// This code is massively inspired by Tokio mini-redis +use crate::GenerateRequest; +use bloom_inference_client::{Batch, ClientError, LogitsWarperParameters, Request}; +use parking_lot::RwLock; +use std::collections::BTreeMap; +use std::sync::Arc; +use tokio::sync::oneshot::Sender; + +#[derive(Debug, Clone)] +pub(crate) struct Db { + pub shared: Arc, +} + +#[derive(Debug)] +pub struct Shared { + state: RwLock, +} + +#[derive(Debug)] +struct State { + entries: BTreeMap>)>, + + /// Identifier to use for the next expiration. Each expiration is associated + /// with a unique identifier. See above for why. + next_id: u64, + + next_batch_id: u64, + + /// Current batch id + next_batch_start_id: u64, +} + +impl Db { + pub(crate) fn new() -> Self { + let shared = Arc::new(Shared { + state: RwLock::new(State { + entries: BTreeMap::new(), + next_id: 0, + next_batch_id: 0, + next_batch_start_id: 0, + }), + }); + + Self { shared } + } + + pub(crate) fn append(&self, request: GenerateRequest, sender: Sender>) { + let mut state = self.shared.state.write(); + + let id = state.next_id; + state.next_id += 1; + + let parameters = Some(LogitsWarperParameters { + temperature: request.parameters.temperature, + top_k: request.parameters.top_k, + top_p: request.parameters.top_p, + do_sample: request.parameters.do_sample, + }); + let request = Request { + id, + inputs: request.inputs, + parameters, + max_new_tokens: request.parameters.max_new_tokens, + }; + state.entries.insert(id, (request, sender)); + } + + pub(crate) fn remove(&self, id: &u64) -> Option<(Request, Sender>)> { + let mut state = self.shared.state.write(); + state.entries.remove(id) + } + + pub(crate) fn len(&self) -> usize { + let state = self.shared.state.read(); + state.entries.len() + } + + fn next_requests(&self, max_size: usize) -> Option<(u64, Vec)> { + let state = self.shared.state.read(); + + let requests: Vec = state + .entries + .range(state.next_batch_start_id..) + .take(max_size) + .map(|(_, (request, _))| request.clone()) + .collect(); + + if requests.is_empty() { + None + } else { + let last_id = requests.last().unwrap().id; + Some((last_id, requests)) + } + } + + pub(crate) fn next_batch(&self, max_size: usize) -> Option { + if let Some((last_id, requests)) = self.next_requests(max_size) { + let mut state = self.shared.state.write(); + let batch = Batch { + id: state.next_batch_id, + requests, + }; + state.next_batch_start_id = last_id + 1; + state.next_batch_id += 1; + return Some(batch); + } + None + } + + pub(crate) fn next_batch_minimum_size( + &self, + min_size: usize, + max_size: usize, + ) -> Option { + if let Some((last_id, requests)) = self.next_requests(max_size) { + if requests.len() >= min_size { + let mut state = self.shared.state.write(); + let batch = Batch { + id: state.next_batch_id, + requests, + }; + state.next_batch_start_id = last_id + 1; + state.next_batch_id += 1; + return Some(batch); + } + } + None + } +} diff --git a/router/src/infer.rs b/router/src/infer.rs new file mode 100644 index 000000000..2a7aa0ac6 --- /dev/null +++ b/router/src/infer.rs @@ -0,0 +1,130 @@ +use crate::{Db, GenerateRequest}; +use bloom_inference_client::{Batch, BatchCached, CacheEntry, ClientError, FinishedGeneration, ShardedClient}; +use std::sync::Arc; +use tokio::sync::{oneshot, Notify}; + +const MAX_LENGTH: usize = 128; + +pub struct InferError {} + +#[derive(Clone)] +pub(crate) struct Infer { + db: Db, + shared: Arc, +} + +struct Shared { + batching_task: Notify, +} + +impl Infer { + pub(crate) fn new(client: ShardedClient) -> Self { + let db = Db::new(); + let shared = Arc::new(Shared { + batching_task: Notify::new(), + }); + + tokio::spawn(batching_task(client, db.clone(), shared.clone())); + + Self { db, shared } + } + + pub(crate) async fn infer(&self, request: GenerateRequest) -> Result { + if self.db.len() > MAX_LENGTH { + return Err(InferError {}); + } + let (request_tx, request_rx) = oneshot::channel(); + self.db.append(request, request_tx); + self.shared.batching_task.notify_waiters(); + match request_rx.await.unwrap() { + Ok(output) => Ok(output), + Err(_) => Err(InferError {}) + } + } +} + +async fn batching_task(client: ShardedClient, db: Db, shared: Arc) { + loop { + shared.batching_task.notified().await; + + if let Some(batch) = db.next_batch(32) { + let mut cache_entry = infer_batch(batch, &client, &db).await; + + loop { + if let Some(entry) = cache_entry { + let mut batch_cached_ids = vec![entry.id]; + let mut total_batch_size = entry.request_ids.len(); + let mut max_sequence_length = entry.sequence_length; + let mut request_ids = entry.request_ids; + + if total_batch_size <= 16 { + if let Some(batch) = db.next_batch_minimum_size(16, 48) { + let other_cache_entry = infer_batch(batch, &client, &db).await; + + if let Some(entry) = other_cache_entry { + batch_cached_ids.push(entry.id); + total_batch_size += entry.request_ids.len(); + max_sequence_length = + max_sequence_length.max(entry.sequence_length); + request_ids.extend(entry.request_ids.into_iter()); + } + } + } + + let batch_cached = BatchCached { + id: entry.id, + batch_cached_ids, + total_batch_size: total_batch_size as u32, + max_sequence_length, + request_ids, + }; + cache_entry = infer_batch_cached(batch_cached, &client, &db).await; + } else { + break; + } + } + } + } +} + +async fn infer_batch_cached(batch: BatchCached, client: &ShardedClient, db: &Db) -> Option { + match client.generate_with_cache(batch.clone()).await { + Ok((finished, cache_entry)) => { + send_finished(finished, db); + cache_entry + } + Err(err) => { + println!("{:?}", err); + send_error(err, batch.request_ids, &db); + None + } + } +} + +async fn infer_batch(batch: Batch, client: &ShardedClient, db: &Db) -> Option { + match client.generate(batch.clone()).await { + Ok((finished, cache_entry)) => { + send_finished(finished, db); + cache_entry + } + Err(err) => { + println!("{:?}", err); + send_error(err, batch.requests.into_iter().map(|req| req.id).collect(), &db); + None + } + } +} + +fn send_error(error: ClientError, request_ids: Vec, db: &Db) { + request_ids.into_iter().for_each(|id| { + let (_, response_tx) = db.remove(&id).unwrap(); + response_tx.send(Err(error.clone())).unwrap_or(()); + }); +} + +fn send_finished(finished: Vec, db: &Db) { + finished.into_iter().for_each(|output| { + let (_, response_tx) = db.remove(&output.id).unwrap(); + response_tx.send(Ok(output.output)).unwrap_or(()); + }); +} diff --git a/router/src/main.rs b/router/src/main.rs new file mode 100644 index 000000000..5d87cd46d --- /dev/null +++ b/router/src/main.rs @@ -0,0 +1,125 @@ +use tokio::time::Instant; + +use poem; +use poem::middleware::AddData; +use poem::web::Data; +use poem::{handler, listener::TcpListener, post, web::Json, EndpointExt, Result, Route, Server}; + +use bloom_inference_client::ShardedClient; +use serde::Deserialize; +use std::time::Duration; +use poem::http::StatusCode; +use tracing::instrument; + +mod db; + +use db::Db; + +mod infer; + +use infer::Infer; + +#[derive(Clone, Debug, Deserialize)] +struct GenerateParameters { + #[serde(default = "default_temperature")] + temperature: f32, + #[serde(default = "default_top_k")] + top_k: u32, + #[serde(default = "default_top_p")] + top_p: f32, + #[serde(default = "default_do_sample")] + do_sample: bool, + #[serde(default = "default_max_new_tokens")] + max_new_tokens: u32, +} + +fn default_temperature() -> f32 { + 1.0 +} + +fn default_top_k() -> u32 { + 0 +} + +fn default_top_p() -> f32 { + 1.0 +} + +fn default_do_sample() -> bool { + false +} + +fn default_max_new_tokens() -> u32 { + 20 +} + +#[derive(Clone, Debug, Deserialize)] +struct GenerateRequest { + inputs: String, + #[serde(default = "default_parameters")] + parameters: GenerateParameters, +} + +fn default_parameters() -> GenerateParameters { + GenerateParameters { + temperature: default_temperature(), + top_k: default_top_k(), + top_p: default_top_p(), + do_sample: default_do_sample(), + max_new_tokens: default_max_new_tokens(), + } +} + +#[handler] +#[instrument(skip(infer), fields(time, time_per_token))] +async fn generate( + infer: Data<&Infer>, + req: Json, +) -> Result> { + let start = Instant::now(); + + let output = infer + .infer(GenerateRequest { + inputs: req.inputs.clone(), + parameters: req.parameters.clone(), + }) + .await; + + match output { + Ok(generated_text) => { + tracing::Span::current().record("time", format!("{:?}", start.elapsed())); + tracing::Span::current().record("time_per_token", format!("{:?}", start.elapsed() / req.parameters.max_new_tokens)); + tracing::info!("response: {}", generated_text); + + Ok(Json(serde_json::json!({ + "generated_text": generated_text, + }))) + } + Err(_) => { + Err(poem::Error::from_status(StatusCode::INTERNAL_SERVER_ERROR)) + } + } +} + +#[tokio::main] +async fn main() -> Result<(), std::io::Error> { + tracing_subscriber::fmt::init(); + + let sharded_client = + ShardedClient::connect_uds("/tmp/bloom-inference-0".to_string(), Duration::from_secs(5)) + .await; + sharded_client + .clear_cache() + .await + .expect("Unable to clear cache"); + tracing::info!("Connected"); + + let infer = Infer::new(sharded_client); + + let app = Route::new() + .at("/generate", post(generate)) + .with(AddData::new(infer)); + Server::new(TcpListener::bind("127.0.0.1:3000")) + .run(app) + .await +} diff --git a/server/.DS_Store b/server/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..09937716d14643b9416026cef19e7ff913d0ca36 GIT binary patch literal 6148 zcmeHKOG-mQ5UkcL0xrzb zHN&gL>ofpc9v-iO1%Nr-5$_(R=Fi0Qmp=ZhdVmDUXH#D_~ z==O2mi1Z@T!Hu%EFfv6xlba0Z`{^=VuNNz%&5~Av_cFTv+DB3WDnJFO02QDD(@`Lg z@@_euN8*!E0V*&N1?>A!;KrJ4LH~3h_y_g`kg4sXJl0o!i5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!HserR!OQL%o0 zPKtg~PJVtagw)N-hwZU{2>--fIdSHOgv5K}BuzhX^q$}I8|U}Fopw4c0@rtIJJ}yTA%CK>dciZY z1Ej~m1QE0#3Eo*jCoEwUWv}oie&SCWNn;WufqUi`%}JPqPHq&@q?NQL?W9eKWIqrQ zh~^~`%?oxxlNj_+M4(3(Ea?ckM+U9G0F8_W)J7sTX;!3K4?OEX%QIucpQa|S%CxZl zkGVE*C_a>>RH>1Cy8q39+1^N=nCI0?S?-RjN=s#Xd#0LB^YU<4OVW1FCd+Ivns^qeo=~T@~*le_gU3W57+i!vT zHxM~{x)O+~pc{uF+E3x88!y#cru9k%Q?$`hKtasO)(V%NX&dYQnIkzp&CzY5LD94;1&(dOU@$%Bx zHO|wrtW1j2ra4y)#Kja^Pa9BhY%!a4)f1Rcn9bk`tf5y-_@Z&?CrufM;1zZA<|iRc zZ3J3Nw80jMSae|eTB0k~;NBKJ=#@vd;c|7WWA2Sf4n4zRTUq4tD$a7xi zhV%E)^LeLt9SSsgZ5l9FWg94IW~$8GDL9T7%D8=Zfi!j_G&p%p&tWlLu|=OU^$A$4 zH?5}EOkqarwz=IR>N7BW1hjA?s@;IMpwl0N;T|%k+#$aS0%Fdn+BnCd1MLVF;xY>m z1Nz`_L3TdiU#=A2&5Ng{!GjuuA~d9Db1&dIc!4c(&kM|f^8%Z{#{TvOS>Y;B17x@3 zwSka@G!jjVkmEY!+VKBnzDe{Itc1HkLcI%_s)y>Wz63C&1z@`dbT8b)Tc#KB>h1fQ zUBUKR6xMSKsIIR6Ql9DuF>%s1S1>y@=6r{a2C;e@tm++P?;`sY*$2q53iY1B=1`v_ z`vMupP>=9UQg5dBDOleF(pZs@hTewP^}8NSE?Nd%^&#ABd->ItT451wU{3M#>()n& zjoZZb-ACl+J29N@yA7B;zH6A7ywEF4#;r)iT&exiO&f02n{69s*iBE!V!8p;^BC=| F{|781%S8YH literal 0 HcmV?d00001 diff --git a/server/bloom_inference/__pycache__/model.cpython-39.pyc b/server/bloom_inference/__pycache__/model.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f0aeff1a277f9b6e70276691afd54342324fa30c GIT binary patch literal 9759 zcmbtaO^h7Jb?*P^>G|2&+1Wpml2)>nU!}Xhui;eiQru{3u%w85=F5-^=9YknC?`Xby>ijf1 zM%VOBop@%)>e{}|X{+OObAGPt`ffMx=ewToG2iYKx<$X(E%~Kx*)KEC>CAO2eudMy z&V0A(S2^u=7P>XR#_4=#vAg6ib(j5Re)l>n-Bo|Jd&WQ0J?o$Cp7YP?TBMC16WX4R zl=*#46h!f^CW^bpuI@k1oRTPmQ)bQ+C^080c(3dl{uW@pA`BPt-kO% zMy+ks^(-&KBy4oTIL3?hPMB=ht;~9}6(^Z}bvWomnfvx2Y4v+yC$lcM8cE&EO5YtO z{j2@!QLnWV$*e}NZ}mmgdA%<$g~K@Py!lq<5%*HR*KBQ}=DA)!X!Tl2a4TwUZ70;2 zcPVUaN8jru@($jLTTw5PVG;#{&6ny%R*1JlDS|Fln-#lZ(%25-VYAt~js8`xB>lml z)!VuxTS+8aVOH6QZYSKOOWXZ^j9#6i=2~F~T?~@G#ODHolC8~Q5{Ya+j*>t`&2ZSk zGr2Q}8YGd~%fZV<-0>JB(tKTLzVR+*AhLvhL5u9jT-A^^kanV6=^f{bnqqc{m#BF^%y-e9PbrLfb%WE{LM^dLHj2-6ufl#5r|Xnm(Pr%g^fV?D8uW49fqIi@btywLAiTq=iB9(V;|+|!A3 zV<|OKTbRNU_C0f4Oi|k?pXP3w3c+>t7+|mTgE;JBQ((cCyW#Di7u`~liy@ojYTfY5N)|y9_2Ryav)mvQ0xMN6 zLk6GkkH;4mUw9Xr zCw`&d5ie|Z`u(n=&$oKbNJhOz^a8SPMpCh{iVoPkQP&4|vQks_yMbDSgH7si{0fL> zSvvnrPcPzU>4xs8-(&=LOEaUH`2e-^Wbr^u`fzd)ce;g;lkgyRb)jE3Nu0YQUbTD^&Mn^cIgEBTVNZQIZAA z_^Zo9j%2zu7k@ta5MgoI}6Jgq$HEDC-3aCbl&y8N{o%|X8)MJrvYY3Y(E?$PQv z>@%;2gEzExP?F~0NJ*Id`}&Pvqc`PmX`{l8+bQj)l{5$Q;gT6)6H?mvT*tPRM0WDL z;n&`{ytcEj*6Sy0&Hk__)>^$a-u`uy^>0=V6IctvSV1?8uV-_E5JvlYbSJnOc3{98 zRN28r+KrS%OgCxuRJ&Fl>7*KV_4hPbKTOvZO_I)ql`=a~4;s^0KcCr3Umnw6`3nTR zZV%+EZG7<`xV4?YqY3V>05n^_(CIhAPW6kos9=0Z1ihHcLZUgYQy_*NCv)U^9EH=|Y@@63DA3qT89?;vr_95(SOR z#}LFMd+zw~^tNx&a`5v9iz4<5oK$Ng(}T?2jzR%++x3?SY((OukRYRgO2vL_Ct)-{TTrMvr>Gyh$TGY~6?68jW}%v+@Z-!^?}Lr{QsxFh4-hQ~GA|%U zW!Rx~F$iu9!_FkfZ1s|CE(pRN_J2rwJPra$Kv@z-$xOH;@^wl%R~a!f`eSqlm_u$E zFyMAy$P#^0BjOTaQ1=q0ibQC~_yz9nCBjBRUM2Q)uXBn33)ev_G$wH987`)#s+Vbk zAQw+J9+jUm^r~m*T(W^eM=sJm7jeggh)(f}$U7pB2$)@%4|KxdcP&3B2w&&2>g({z zV)XC(%G-lHGH>dpWKMsWz%Qxm3f9dkK1!3^E>YuEkVFHZ(*WYNwh3pzKm*1Gz!dFQ z8GzI+CADkB_`^dk+}t?oG-rgvKn}_Q`t2W3EG8YR={qYYJG;(>HtMc?m+E_u$d8El zM1D+!CyJX4>GiZJpxnR59g~YmQ{WnscqfEygLx5m`~#2!VMiD(1ZxNrg0(~;vVq}r z#!R-)j%bvRfK=e1!mbfQ5iZ=tyzo%IC@jdS2xsxlx8Ht?yl@8YypT_z=tFV8kLF3- zV`C54IMK%F!XAVln7tyu0WQ#6Vzup5C(r91rX_Ha9DpZEj@>;CkPj2Z<=}m_y>_8p zM42L4tGd0BS&g9x;kyimnbnD!$wS>QPJE{zZi%Q~;(a0o`jQn}aS+}NTb*#T6RFKW z+ajA|MKypACc{<_+kibF0@0ZjLAS~0C~5PUedypgQV|Au1$?ZX5AcvQ4Kwq`s{?zT z!xzCnAu}$6XwY9}Xs3!%gLc{}vq0Amifug9+5R<1h6fj=Y07yC(_V%gXc6o1S0PdO zuQWO4#?^G`%BmN=ypB(^T(d84Aw+^z{sG>PeQ>P*a)2WLf;%R4p+QY6 z6>5CBQ&Js2s10_UP18xgid-ZTvWYu>33~)HukUG}J_WBG+pwv%w2v{vztTU^N5+o; z@+|yz5kc!!Uex0A=YdV=_hZ~}G0ZLPXF8sTF@eOw*LG^O9XxZmUEKLydz6EBpCbr; z?Qa2~-S_QLe&me`?2Z)DQtFKhv>GR{k|Fu7F)E{XMdX$V%EEC$dRLeKjGiLml9tAE z0(`YnM{}t&t|apSadW#2d~Xa=b6iPXa!@|j$5mkq=dPA40O)D$8epU=a?3Poq<0N+ zQrNAr@J*RhHd5@~P)+ys_!U4-Y{e_sir7a*a$ex37%g^f09XG8V>QVwNfA>5$c4l4 zk>1o1`El;LsUeE1h?VT>$SER6Lr#fvOyZ8_nFF^8WfiB&oVgj!Lhrf5Hn@IOtEv$> z_Glh7H>!fVqXq6I%B|8YFH(=-4lR-EA#zQ~2yTMnt?cPrhRD%qhy%B06*Zhm=hI4R zr*J=Z9k?w{TBFg|D;tcA85LfwBcc^wZw-PN(KkfuCXtj=;V$jZxB{{Tv0rmX*ff7) z+`x93E_7^+@1THuhsadz^7i#EbP)V&^t*#sS7Aux@1e}jGiv1@1|B)EH3M)DKat<0 zDhc2{1fmAelC&|8^piFPY3l{t@fryCa|H_)n<3Rd)-bIG(WY^t-zN7vvw-*!5K;&X znz_o`Jz$^)@Q6TLtcFD+k`m+@3?*izVmSLs-H??$xX6rAF87I$DfbD^0wxIV!m2jY z6Ifg-@o*x;wMNKD8wbds*Us0 zHURAzf8`LN-jbnURmqc`nLDikizRb9{jF9KPaO&sAtFUCp9g{63KKRq8G@n>1g*2& zBp|1p2r}>TGL^Jt*xQQO;AB?6he>^d^2jhLLq-~!orLml zFa88XL*(&1+z4_daGM17C%^x;=A18peGh)-IkRlKu#(=sV^#6pvoQ8WxC^dX)g1sf zW8XF5>^$-;I6ZiCPzF4>F+~;AG%b7$2WH=RWH`oudBD>Y@tpj=u-u*6Y1*fbU}Wi3 zY}vs8B!LIDKG_o>g`<24US7O#J|x~$`*DiIA~!+)5qC`IVy<2;R?w&FBjz{2I&zWP zpeud_WC}d#+r zz?!?$AjNrb(Etdm5ic%71teMmT)B%t=N?c)V(x11o4||~ZkzBWEHHvu*9mLp5}db; zJ>b(nNL(NSatrKVXupK5LN0=t?c&JnRXIRj?0khIs0o7B%cRBe0Xn;_e zhLZkrTb+{sgV}uU>l571@;wT`y5qIf21dfU%J|7WbtVI6VdN!G;mibx2rYbq z(AP~-0R~$_U>N+gKz#X`D2zPheU^gCn-mB`2<>Y)an$#W@z?i&l+rv0&Tgi;@i!Er zLh5m}kFfCgX_U}{-%6eKGt{GTeNQ_>oM(^slD&kgLJvDOxeDKr5tfN;64?U5*-I%I z1h@LKg8&GfPf={=gax5J!(f7ZU^a*q6+I&?yE_Xiuw==r%iM4??ssr%2nGR+3;~&B zduEUE_9A;MrdDOAKD+u4-GbB51wvwJh5clx&ioKyzU?XZ#f1q_VN zgF2n4^EvGf*yf*8R~&VmTDNw-K|XD87)e}5luX{41`*d)!JV_miU|ZAqJzvDzXjV+ z=fNx6$GERB`+&=ksQ%jsmkn`P#!5u8iXzWED-C2Ha3zL8#gH-!ChvOYO{XSv5a@*2 znR;|`8{erwoV(fV^uy%EZ^Ff)#LF+KkVlJ3us=ucS5}xk&RyJ;{ZKS0Zra~$L$w41 z-Xy6y_9B&m!a*n60~M%Y1WJK1Ii|aDL*1YE7@OCs&iTimPE20`-`XYGY8G%>|HsF-t?=? z1kOYppdTD|6+yv1N1%bh)fqc*ZaxS&{)BMV)QHd&|BzNW?F2g8Vn z8e(%WnjBekts;z|OU4l~Jcqbe5oyE#jd$QTd-%q}_m<&0MerQ`9P+C8?JSqL3sCdm704YF?mYtzr!8`6ZqFoTnpy!4n8_4wT$Md^ zjlG^z719ZWPOkOGIMtg)=!U|n9!IaSzONrBB>6*BPT>*-QUhKOGm}QD>8~(pvtG{? zWtP_`ZvSa|N#3JAv+ENh1zbXL2L(R}Aj&@{@&*wSy!;~~uM_!G5Z+e77FH;>`F-M! zh)^J_o>%Le94hvt)Jefl>2(9duiV5L!=@SstMbg7aA=Eg+q@+In!aG|J3#8sXkC(i z&?w5QeqTB{T2mgNSWRES3Y);LW5Xxdl|L1B(YjKK?GSrqrOD1{L_ofi-NF6&ODfF$ z`D3Ksq5cq%Ld5$GNu2sqY6|E19AbS|ffWw_h69IyoFDq*K(gIwZA$7w=HM?X^e+Oc zPm&E&Sde4LMc&AmNt}bkN>|dhP}=Y8Z00%#zEDpvOj`Jp0I9fakNYEM!ye-1IJ-dDj^QnXT7s=C zycvVb!X`Oq?gg}nxcjS+wH7}O!bb*BDl7OuK@ke3h{K1KoCmaD`HA)ds9XMtrvEQ6 C84{xa literal 0 HcmV?d00001 diff --git a/server/bloom_inference/__pycache__/server.cpython-39.pyc b/server/bloom_inference/__pycache__/server.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e3ffb8e5e5b523a5b6787ee24c398b63cccea77e GIT binary patch literal 3683 zcmb_f%WoUU8K2n~mrsc@^>EN8OprDqQcF&n<)m6vuj~M}}jlu@Rb) z?5OAz)w>mzqOwy~<3d=8s!lbkIkjlbSyOX%xE|G=dbHteQ1S_3MOHc`ti*}@PjWF(D-bzK7!NbkM-{Si%EePI+<^#FAKA|@ z9BXjz!<}vc8EGTpk`V<;M>hx~^B(q(Z4y5ZM&(s1!s0F{__Cn@~d!%S(LWi%dh`ze_k_z~pQURHR z_+(0Fgc7Mu^%;0Z)M2`1DBpm$L+%NsOlBTLeR(3#r>g>%_Hmm6H!i0N5VS;xS675v z?7JWqdIFPx66+BRBnX9&>1jQH3{YdHkJV^uj`SJ*oi;YcCahS~!pNLa5Z~?<8D-ij z85wuTgpN$n9+}h9jKlyqrhlt}hm5vQ;q>{1mOxPt;ZD)gM_`ZNSH{44N@nCQ+7kop zfyO?A*p&%{g7G0+m&7fd}{~AW^`%6(b|K<`l&~yqLZ5S-DG1C+#^r>+H6B=M7$Ar}@ ztYG3UV8Z@sFd=>p!id+9TvT4X4s+MwUby`I|Kakl0(q>$=4HMYZ@?L?V(#b0lAjd} z7`l6rK!1sHyoCfak9Zpi`ckxzTof>?&n^9eKYn@crrN5Tn*UGucB_J^HTELzx>?C} zp;Qb)99LZT$-oQq6=9<&1tb{onVw1^Hu3c;lAj@Y2g$FHEJRi#M3txl!8^r%!lqlM zt=ooORr41>qhbyA?BnKLAk36RQmIKD%Ap43P-n)eIo73t31x)!Nt-}o!5#q;h1I?b zC9kAPp3*BTqmC4kY{om%8sV&tvrNs6%p=$(3kc`2C5zLN zERS?qnLq-Z=p$6K9-Umsc1Mjk3K+MJPlp#7&< z3$|DRxC5UFSOnI#7i-Esc6<2hAP&C%Xm4*%M*Y=FeC?a@@cN3Kn$aMY%?@uu(rG?B z=5h1fqRk*}J`@9fDK%q7K;Nr2LZ#-~n1TVopfkFZoT0I@x`~K9+0sN4KFAE=#gEUZ z(?A*BH1QvE=}I2;LujfNSxN=!E#!ce0NQ|EsLW57^00b^mI1o4mj);n69v9u1IsQK z%~X0qa@O=@^P6#NCo4Vj!T~=JA`t>Xo0%Zm5zk7`5)m>t#U%Ycp7H?_6(A@}W+~P} zl;OpB(__rN7=*|U^15)LDxGh|z{sOZ^&X`=2Z#F)?tSs_{sU(nx4`z)?VKn$JH_rG z4AoHR$f9z1@ZizC{R6js`||^5hsRzA0ysa}U39lj9YqPCgYgFf+S5eH%v3sOW?uxc zbQ;e*Xt2N?U~drxv8S41TfB?Xy@%x2NU(iUO^{PmMGqTE@excq)n!Qzb7y^syH#Du z`_9}YIGbJvLQI1i)P`RPe)e-quW2~8fEu*++%PKCfRa;zxf=Y2uPDV9E>*Ajp)Kuh zyJZPHHZwX&5(FU0zc% zyrgwx0q5)WypFD3F8S!i1zL(B)r`WOhHJEPg-Z?*$_f#P{xpL^IDGJ-1-I z2Scm?DUd`3KE`#h;&XBX=0nhMfwfR8W@(owTe3r`95L$mqr(ftYq0I>_cuQljEl`I zrklHQmL(-!kJ3KlEDhNve161ux$G}24+2j8M1$POk5nTGWH|6k;jTty;9_2fdDm8+ z;q=oWVam?ITGf=f|7pf!>WgU1RE5f;oP{>9&z~ij=wN9>#R2O)3yFasm#!-0u{X zv&uP|RHai?&#LEIrdzmo(J9{f$CpHsDKWCDQ*iC1Cf&!Rk7*4wy3{K*o?^aM)F375 zz~L3%)LKEbJ|yC=AE4i3{eBZo>eAb<7qy~J>$Cw#8nRJ9qD(E|`={oUP5pis?Y#Xv zGHFg)MXQupfTU9_93!iB);dRL?Q>K(Me_!lk}Y)b)hSMwX1JOLSL+hj-wduc;A$7` zIWCa)&`5)o?;vYGj|@0J`wq6BFJJ?yGi-nPZ`e-p!58VH8McnWc5Q}@@K#Z~vPSz( zTO$z#RL2DTusUf<@2peQ&heGCAo1VnkLU5*xjAVrN}2oQlpL%S5V3dAXSD+^oAO#= z7tS$666D^|Ho)jit`(jArTy!W8xCE1LNEe-%LS~S!@RK1UV$0p5lH($=Ti6j`1Rl1 z`|!j2YXZ_W>PO+4(4`Qidus^`2SFN%B*gO^hJRfMy4J445I+U72i3ZBdxEEUivEZw z)GAOsB|yoaom=P+pVz@>tlO8>jT}$<&h~|sAsbAwyHuAX@;)F8z7lC zHG!sARyWs;PzFJI+pP zf(v|x&e10bqXX!h4-l`wcj!3|I|QYoOLzwsR80hrn6K;Jpf8C*SKHcSDLc&hwl4So zz(C{MF9$Yf9KU)m3yi~qn6rQ$t%X52V07(K5RaI62)?Pyj6*vzJwZS^zX^kD&%7XBd;AwL#3~SoeP@Cpd3C=g z&w1{mY?v2ww}nAUBN|9{ft4pDOiT;C@*jT%M6PMsT8Rx|?K9}|LFg)?M8tZuO@N6L0Kx8Kk#Md-x$Mv8vM z6UH~7!&9FnyNv39iP_$-TXEtKAat{LuIojgR5W_AcD}#&xCy~6ki}osL|0uIAMGU* zflDu0<6F;}Z;YH9w_x@iWB1%_{u+pSs`37SF#m6$=1{)`O2#){3e@;|AqI1pdL1gD zng?)xOQBp-iGDWhuBZw`W{)W=h+|dBd6Y`7^{95AXNjM42BC{mM(4Rkqfjbaw-04Q z`Uu#W9;phHXAUQ}sWEuV<4AISP^%hCM-X*LQ{_-5p(<4+jI)#}Z_Icm{5Tqd`f-#i zr^Kf!$&f}|*}6d}C&+V_((cmp-K-7Ld){Re5wK0)JbFu}QwYB-4GmZDH-VSGrQuY} zvZ`kznU7?7&+y=u6AsP1*EYKu^~-_~{93t7_jBb%>3kDk*6KTDSGrPV@~dkvV7;wX zoAzsZTfN+2pz8K4VjzgLi1UqOFJTwk#Kz9Ea^Aomthl&A$Twcy!yTA+%x@XrELU9* z>s=k79elZ3UB)fEYBlgGaj>^|#Q9o)bz!nZ-6a(zF6u=@VEvm5P+Ff#W#bK4s? zHle!&o246{>WdpRIb(r#dAFnUNDks?m+NDUuWO=@0o7Qja$&N_T=kSCBv+QsEG^&? zfWG+oe(5UhO`}7(yZtx|G~)6tMxjWmx9 z$(3Vabg6tQ>U)pU(f<;!Jrut6Toh=3A4hAg<>XQ>$mRFVNAi3A$aH0;$Iw2$vX%X& z&)7eRSZ+RuEp+`?41x(>uu1#RC*0BA#B=|?`}c~V3@2e3O`@_h>6Gy#=Im1@d=Wfl zA~^9TT~MKjKt-T>pgJN36@%)7>WUty9;kuni@{SiS&@8yC|1PqDLdhlD}p~@qbo0< z4;zmHd;NYr&5iy-)tS_~R`+vl?5%BC&ZJ7sT*>zP!&HG`SGVig7q4+XU*=@A^K$b* zY@zEL7$hURVJjCNbn}G|Z2}uVNXuD~SJRB77cIbq`dxJW6AXs!^L=mMZ#b=g<{f+B z1dT`jdFGvl!aw83enWeP;}`!$d(N#d^6~}8>Jhj7YF;Lq&XRf*+0fJ~+p}I(+1~EF z%1mBYX<<9M-Zf?Vwe6Sl!sJz5=8pT-T*@N}bptJnT^kk7bvB%-dPh(CmHgTyraq9B ze$Lc)aN@tWHvXuk(i?RlHg<}-F5B_lyxNsYR+-$u=R>Jlc5=aTAW6QOr$zfk-NGJDy`|wM z@o4uIbWK5sxugHDa7LdhDaDglf=PQ)Ik;dBFlqQc3IbuIg!IR%1D=X8AWl)LrU<~u zQ`d-lod$AfJ2+*@3Sv2B89%zxo_XO7^~TEumzKVT50s1xJBfbHPD;Ej5_%iM%e0L} zAi}2{i5rOy5;zbZwC~#0F@nEMvA?@l*9huAu?p>9AC2#2VLNvtSxK~W!{0G6?1UYM zO>i6`tJ$2ZJCxlAzPe+0!%z4zZ^Bd0Am>l`8RtxRkOXEx@)2b|Bp*2ms)OL!g^cfH zn#RX&Of^m>1QjmHgbhogCQ^ShofB3CCO%={bOnP^(O7*C`faQ8+Cj0*k^= zqLA8fx2RKth|OlQ5{g2s$UsF7w4KKeIJCiO%52L_^YdpU9JXE)_Op!y>cWL-B4c}V! zl;E^Llfg0XU01}(&|hifo-OoO3!)>CgRUT`FsL9RI3u>J50EZ1#OtogjZ}I1f-3Gc zM4%LMK3>HH{$&X;0V7JG^{Gof&klhGNhKej5resCI|G497SVkca9S1b_fGp~@Mgnz>GJN4ZSdvCNF1A$VSy;_V0*3T-jfbnnhmVN8D?F|A{*49^^8rxkaNQ=%Q2ZXi~%g{La`h$3K9O z!h3W>us<=*9@00jaW_ykxR`I^#fzZwF|g6O@#?aX!aSKtd$sjjYqw*KJR7>nq_5LM zDH2CqGFtr(8a*P_sOp#p^nxw1=lG*l^-C~qFRLqq>$%8x&d+!5SzmI`gyl@+~y@fu9pH(qicICtf>cO2DEd(lx9xvS0LrmHmPM}Sm&aG6%=R4S*`k^wKv h)yBVSpK8A*&ufZijN6NTZ^XmSFdV))9K^Q+{y$$P-bVlc literal 0 HcmV?d00001 diff --git a/server/bloom_inference/cache.py b/server/bloom_inference/cache.py new file mode 100644 index 000000000..5ef1f1dae --- /dev/null +++ b/server/bloom_inference/cache.py @@ -0,0 +1,48 @@ +import torch + +from dataclasses import dataclass +from typing import Dict, Optional, List + +from bloom_inference.pb import generate_pb2 +from bloom_inference.utils import NextTokenChooser, StoppingCriteria + + +@dataclass +class CacheEntry: + batch_id: int + request_ids: List[int] + input_ids: Dict[str, torch.Tensor] + all_input_ids: List[torch.Tensor] + next_token_choosers: List[NextTokenChooser] + stopping_criterias: List[StoppingCriteria] + + def __len__(self): + return len(self.request_ids) + + def to_pb(self): + return generate_pb2.CacheEntry( + id=self.batch_id, + request_ids=self.request_ids, + sequence_length=max(len(entry) for entry in self.all_input_ids), + ) + + +class Cache: + def __init__(self): + self.cache: Dict[str, CacheEntry] = {} + + def pop(self, batch_id: str) -> Optional[CacheEntry]: + return self.cache.pop(batch_id, None) + + def set(self, entry: CacheEntry): + if entry is not None: + self.cache[entry.batch_id] = entry + + def delete(self, batch_id: str): + del self.cache[batch_id] + + def clear(self): + self.cache.clear() + + def __len__(self): + return len(self.cache.keys()) diff --git a/server/bloom_inference/main.py b/server/bloom_inference/main.py new file mode 100644 index 000000000..b81ed0401 --- /dev/null +++ b/server/bloom_inference/main.py @@ -0,0 +1,30 @@ +import typer + +from pathlib import Path +from torch.distributed.launcher import launch_agent, LaunchConfig +from typing import Optional + +from bloom_inference.server import serve + + +def main( + model_name: str, + num_gpus: int = 1, + shard_directory: Optional[Path] = None, +): + if num_gpus == 1: + serve(model_name, False, shard_directory) + + else: + config = LaunchConfig( + min_nodes=1, + max_nodes=1, + nproc_per_node=num_gpus, + rdzv_backend="c10d", + max_restarts=0, + ) + launch_agent(config, serve, [model_name, True, shard_directory]) + + +if __name__ == "__main__": + typer.run(main) diff --git a/server/bloom_inference/model.py b/server/bloom_inference/model.py new file mode 100644 index 000000000..a762a7929 --- /dev/null +++ b/server/bloom_inference/model.py @@ -0,0 +1,428 @@ +import torch +import torch.distributed + +from dataclasses import dataclass +from pathlib import Path +from typing import List, Tuple, Optional, Dict + +from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig +from transformers.modeling_utils import no_init_weights + +from bloom_inference.cache import CacheEntry +from bloom_inference.pb import generate_pb2 +from bloom_inference.shard_model import shard_model, match_suffix +from bloom_inference.utils import ( + StoppingCriteria, + NextTokenChooser, + initialize_torch_distributed, + set_default_dtype, +) + +torch.manual_seed(0) + + +@dataclass +class Batch: + batch_id: int + request_ids: List[int] + input_ids: Dict[str, torch.Tensor] + all_input_ids: List[torch.Tensor] + next_token_choosers: List[NextTokenChooser] + stopping_criterias: List[StoppingCriteria] + + @classmethod + def from_batch_pb( + cls, pb: generate_pb2.Batch, tokenizer: AutoTokenizer, device: torch.device + ) -> "Batch": + request_ids = [] + inputs = [] + next_token_choosers = [] + stopping_criterias = [] + + # Parse batch + for r in pb.requests: + request_ids.append(r.id) + inputs.append(r.inputs) + next_token_choosers.append( + NextTokenChooser( + temperature=r.parameters.temperature, + top_k=r.parameters.top_k, + top_p=r.parameters.top_p, + do_sample=r.parameters.do_sample, + ) + ) + stopping_criterias.append(StoppingCriteria(max_new_tokens=r.max_new_tokens)) + + input_ids = tokenizer(inputs, return_tensors="pt", padding=True).to(device) + all_input_ids = input_ids["input_ids"].unsqueeze(-1) + + return cls( + pb.id, + request_ids, + input_ids, + all_input_ids, + next_token_choosers, + stopping_criterias, + ) + + @classmethod + def from_cache_entry(cls, cache_entry: CacheEntry) -> "Batch": + return cls( + cache_entry.batch_id, + cache_entry.request_ids, + cache_entry.input_ids, + cache_entry.all_input_ids, + cache_entry.next_token_choosers, + cache_entry.stopping_criterias, + ) + + @classmethod + def from_batch_cached_pb(cls, pb: generate_pb2.BatchCached, cache) -> "Batch": + if len(pb.batch_cached_ids) == 1: + cache_entry = cache.pop(pb.batch_cached_ids[0]) + if cache_entry is None: + raise ValueError(f"Batch ID {pb.batch_id} not found in cache") + return cls.from_cache_entry(cache_entry) + + total_batch_size = pb.total_batch_size + max_sequence_length = pb.max_sequence_length + input_ids = {"input_ids": None, "attention_mask": None, "past_key_values": []} + request_ids = [] + all_input_ids = [] + next_token_choosers = [] + stopping_criterias = [] + start_index = 0 + for i, batch_id in enumerate(pb.batch_cached_ids): + cache_entry = cache.pop(batch_id) + if cache_entry is None: + raise ValueError(f"Batch ID {batch_id} not found in cache") + request_ids.extend(cache_entry.request_ids) + all_input_ids.extend(cache_entry.all_input_ids) + next_token_choosers.extend(cache_entry.next_token_choosers) + stopping_criterias.extend(cache_entry.stopping_criterias) + + batch_size = len(cache_entry.request_ids) + end_index = start_index + batch_size + sequence_length = max(len(entry) for entry in cache_entry.all_input_ids) + + if input_ids["input_ids"] is None: + input_ids["input_ids"] = torch.empty( + (total_batch_size, 1), + dtype=cache_entry.input_ids["input_ids"].dtype, + device=cache_entry.input_ids["input_ids"].device, + ) + + input_ids["input_ids"][start_index:end_index] = cache_entry.input_ids[ + "input_ids" + ] + + if input_ids["attention_mask"] is None: + input_ids["attention_mask"] = torch.zeros( + (total_batch_size, max_sequence_length), + dtype=cache_entry.input_ids["attention_mask"].dtype, + device=cache_entry.input_ids["attention_mask"].device, + ) + + input_ids["attention_mask"][ + start_index:end_index, -sequence_length: + ] = cache_entry.input_ids["attention_mask"][:, -sequence_length:] + + for j, past in enumerate(cache_entry.input_ids["past_key_values"]): + # TODO: this could be done without the views by using indices + past_keys = past[0] + past_values = past[1] + + _, head_dim, padded_sequence_length = past_keys.shape + + past_keys = past_keys.view( + batch_size, -1, head_dim, padded_sequence_length + ) + past_values = past_values.view( + batch_size, -1, padded_sequence_length, head_dim + ) + num_heads = past_keys.shape[1] + + if j == len(input_ids["past_key_values"]): + padded_past_keys = torch.zeros( + ( + total_batch_size, + num_heads, + head_dim, + max_sequence_length - 1, + ), + dtype=past_keys.dtype, + device=past_keys.device, + ) + padded_past_values = torch.zeros( + ( + total_batch_size, + num_heads, + max_sequence_length - 1, + head_dim, + ), + dtype=past_values.dtype, + device=past_values.device, + ) + input_ids["past_key_values"].append( + [padded_past_keys, padded_past_values] + ) + + input_ids["past_key_values"][j][0][ + start_index:end_index, :, :, -(sequence_length - 1): + ] = past_keys[:, :, :, -(sequence_length - 1):] + + input_ids["past_key_values"][j][1][ + start_index:end_index, :, -(sequence_length - 1):, : + ] = past_values[:, :, -(sequence_length - 1):, :] + + if (i + 1) == len(pb.batch_cached_ids): + input_ids["past_key_values"][j][0] = input_ids["past_key_values"][ + j + ][0].view(total_batch_size * num_heads, head_dim, -1) + input_ids["past_key_values"][j][1] = input_ids["past_key_values"][ + j + ][1].view(total_batch_size * num_heads, -1, head_dim) + + start_index += batch_size + + assert pb.request_ids == request_ids + + return cls( + pb.id, + request_ids, + input_ids, + all_input_ids, + next_token_choosers, + stopping_criterias, + ) + + +@dataclass +class FinishedGeneration: + request_id: str + output: str + + def to_pb(self) -> generate_pb2.FinishedGeneration: + return generate_pb2.FinishedGeneration(id=self.request_id, output=self.output) + + +class BLOOM: + def __init__(self, model_name: str): + if torch.cuda.is_available(): + self.device = torch.device("cuda") + else: + self.device = torch.device("cpu") + + self.tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left") + self.model = ( + AutoModelForCausalLM.from_pretrained(model_name).eval().to(self.device) + ) + self.num_heads = self.model.base_model.num_heads + + def forward(self, input_ids, attention_mask, past_key_values: Optional = None): + # Model Forward + return self.model.forward( + input_ids=input_ids, + attention_mask=attention_mask, + past_key_values=past_key_values, + use_cache=True, + ) + + def generate_token( + self, batch: Batch + ) -> Tuple[List[FinishedGeneration], Optional[CacheEntry]]: + with torch.no_grad(): + outputs = self.forward(**batch.input_ids) + + # List of indices to cache + cache_indices = [] + cache_past_indices = [] + + # New input_ids for next forward; keep in cache + cache_next_input_ids = [] + cache_all_input_ids = [] + + # Finished requests + finished_generations: List[FinishedGeneration] = [] + + # Zipped iterator + iterator = zip( + batch.request_ids, + outputs.logits, + batch.next_token_choosers, + batch.stopping_criterias, + batch.all_input_ids, + ) + + # For each member of the batch + for i, ( + request_id, + logits, + next_token_chooser, + stopping_criteria, + all_tokens, + ) in enumerate(iterator): + # Select next token + next_token = next_token_chooser(all_tokens, logits.unsqueeze(0)[:, -1]) + + # Append next token to all tokens + all_tokens = torch.cat([all_tokens, next_token]) + + # Evaluate stopping criteria + if stopping_criteria(all_tokens): + # Decode all tokens + output = self.tokenizer.decode( + all_tokens.squeeze(-1), skip_special_tokens=True + ) + # Add to the list of finished generations with the original request id + finished_generations.append(FinishedGeneration(request_id, output)) + # must be added to the cache + else: + cache_indices.append(i) + cache_past_indices.extend([j for j in range(i * self.num_heads, (i + 1) * self.num_heads)]) + cache_next_input_ids.append(next_token) + cache_all_input_ids.append(all_tokens) + + # No cache is needed, we finished all generations in the batch + if not cache_indices: + return finished_generations, None + + # If we finished at least one generation + cache_input_ids = {"input_ids": torch.cat(cache_next_input_ids, dim=0)} + if finished_generations: + # Apply indices to attention mask, past key values and other items that need to be cached + cache_input_ids["attention_mask"] = batch.input_ids["attention_mask"][ + cache_indices + ] + cache_input_ids["past_key_values"] = [ + (keys[cache_past_indices], values[cache_past_indices]) + for keys, values in outputs["past_key_values"] + ] + cache_request_ids = [batch.request_ids[i] for i in cache_indices] + cache_next_token_choosers = [ + batch.next_token_choosers[i] for i in cache_indices + ] + cache_stopping_criterias = [ + batch.stopping_criterias[i] for i in cache_indices + ] + else: + cache_input_ids["attention_mask"] = batch.input_ids["attention_mask"] + cache_input_ids["past_key_values"] = outputs["past_key_values"] + cache_request_ids = batch.request_ids + cache_next_token_choosers = batch.next_token_choosers + cache_stopping_criterias = batch.stopping_criterias + + # Update attention_mask with padding as we added a new token to input_ids + cache_input_ids["attention_mask"] = torch.cat( + [ + cache_input_ids["attention_mask"], + torch.ones((cache_input_ids["attention_mask"].shape[0], 1)).to( + cache_input_ids["attention_mask"].device + ), + ], + dim=1, + ) + + cache_entry = CacheEntry( + batch.batch_id, + cache_request_ids, + cache_input_ids, + cache_all_input_ids, + cache_next_token_choosers, + cache_stopping_criterias, + ) + return finished_generations, cache_entry + + +class BLOOMSharded(BLOOM): + def __init__(self, model_name: str, shard_directory: Path): + super(BLOOM, self).__init__() + self.process_group, self.rank, self.world_size = initialize_torch_distributed() + self.master = self.rank == 0 + if torch.cuda.is_available(): + self.device = torch.device(f"cuda:{self.rank}") + dtype = torch.bfloat16 + else: + self.device = torch.device("cpu") + dtype = torch.float32 + + self.tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left") + + # shard state_dict + if self.master: + # TODO @thomasw21 do some caching + shard_state_dict_paths = shard_model( + model_name, shard_directory, tp_world_size=self.world_size, dtype=dtype + ) + shard_state_dict_paths = [ + str(path.absolute()) for path in shard_state_dict_paths + ] + else: + shard_state_dict_paths = [None] * self.world_size + + torch.distributed.broadcast_object_list( + shard_state_dict_paths, src=0, group=self.process_group + ) + shard_state_dict_path = shard_state_dict_paths[self.rank] + + config = AutoConfig.from_pretrained( + model_name, slow_but_exact=False, tp_parallel=True + ) + config.pad_token_id = 3 + + # The flag below controls whether to allow TF32 on matmul. This flag defaults to False + # in PyTorch 1.12 and later. + torch.backends.cuda.matmul.allow_tf32 = True + + # The flag below controls whether to allow TF32 on cuDNN. This flag defaults to True. + torch.backends.cudnn.allow_tf32 = True + + with set_default_dtype(dtype): + with no_init_weights(): + # we can probably set the device to `meta` here? + model = AutoModelForCausalLM.from_config(config).to(dtype) + + torch.distributed.barrier(group=self.process_group) + # print_rank_0(f"Initialized model") + state_dict = torch.load(shard_state_dict_path) + # TODO @thomasw21: HACK in order to transpose all weight prior + for key in state_dict.keys(): + do_transpose = False + if not match_suffix(key, "weight"): + continue + + for potential_suffix in [ + "self_attention.query_key_value.weight", + "self_attention.dense.weight", + "dense_h_to_4h.weight", + "dense_4h_to_h.weight", + ]: + if match_suffix(key, potential_suffix): + do_transpose = True + + if do_transpose: + state_dict[key] = state_dict[key].transpose(1, 0).contiguous() + + model.load_state_dict(state_dict) + self.model = model.to(self.device).eval() + self.num_heads = config.n_head // self.process_group.size() + torch.distributed.barrier(group=self.process_group) + + def forward(self, input_ids, attention_mask, past_key_values: Optional = None): + outputs = self.model.forward( + input_ids=input_ids, + attention_mask=attention_mask, + past_key_values=past_key_values, + use_cache=True, + ) + + logits_shard = outputs.logits[:, -1, :].contiguous() + + batch_size, vocab_shard_size = logits_shard.shape + vocab_size = self.world_size * vocab_shard_size + logits = [torch.empty_like(logits_shard) for _ in range(self.world_size)] + torch.distributed.all_gather(logits, logits_shard, group=self.process_group) + logits = torch.cat(logits, dim=1).view(batch_size, 1, vocab_size) + + outputs.logits = logits + return outputs diff --git a/server/bloom_inference/pb/__init__.py b/server/bloom_inference/pb/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/server/bloom_inference/pb/__init__.py-e b/server/bloom_inference/pb/__init__.py-e new file mode 100644 index 000000000..e69de29bb diff --git a/server/bloom_inference/pb/__pycache__/__init__.cpython-39.pyc b/server/bloom_inference/pb/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a1d54d8ecf2bddf9eba6c7de3fe9a4ae98c02952 GIT binary patch literal 161 zcmYe~<>g`kg4Mzf$sqbMh(HF6K#l_t7qb9~6oz01O-8?!3`HPe1o6vFKeRZts8~Ng zCq+LgCqF+ILh5GbrKJ|7<|U`<1EtGSiy*@B2;qVx{dgcZvm`!Vub}c4hfQvNN@-52 L9mxF8K+FIDsFx>q literal 0 HcmV?d00001 diff --git a/server/bloom_inference/pb/__pycache__/generate_pb2.cpython-39.pyc b/server/bloom_inference/pb/__pycache__/generate_pb2.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..936aca2917ee5f063bbf8244d43713e2612dce31 GIT binary patch literal 2182 zcma)7&2HO95GM7n)t^Ntvf?DRtD=Q-sNJ|NdT3BYmS{&sV!M}W*q-;xu15|`Q&d<)w$KmeKDwT+Wzu)KY z4*n=A$}6c%|I(=3#lLuiz!XzqDoZ%(C}Ad5DJPxOC~amUIfZi8%tdk<<-AE$1;|O^ zg<=*VC+{aEmU*F?Wk{Y>pbT?AD=!kbCV5hYGRsCf=7r368p}OR`2Vszl<7PpaD`rF z1;iRF!UD=A#2PC@9pwsQjm^O|l&gp}rokf0^N2Ne1+Jrf6|u%@a0BH9#2Tx^O_Z-8 z*4QE}p?n>&#%{ne$~O^fEZJ|EhF*tdKS{5{LJMi786 zw0TkpV1za+82fdDAkC3(iFutuH(^%RYot7~o>3QmpaFjhu88WE z8efp~rWFhhXR3Zka(+xk)=jP!Hm=Onm@sYpND5JK%NiU4n@NFPnWUgLi?gJ(I+^Rs zVssD%M4lw(k30bnEQiK^VLy)>Q_`-Jc}ZMIKJEah1MV<5tfysP8y^#G%XV#n)h8Ru z=I#s+Y#<*8*ul7gjV(eVwz?bm$1^K0WxPNt7c0wGSZe&FlHA!=xlM8h6B^NWDQqmy z^y?Ciw@D#t0tHDVB3Ia0n29>)c~AL8RX59|7&k7CX=F)SA9=y?#;*xd?!mKQss+1b z?&8MQw3+Nt=;8-6wu>RB57{zI#WOekMBbpM=HXWCBpHzp=IRdKpK{>um`g3BO5riC{0Q2=7Vw zR>Bhre@XaTLM4Gvm9Q_tln_ezQ^G3=|44Y7#L2Z1W+`QSKxtSXaz1n*9zQta)@&E& zhHE)Ku2#Yvos77!a$%v5iIlKF&nKOa15F9%rYzCy^W_%>%@^ihm~F#>HFg5zf-cJX zVYSH^zGT~S@PM%`+kud@^=`}P?DckyaA{Lc+Xo%TaK74 z>~U|(i|Z$mBGl-^u1;I@Ja-o#lzRalPhno8UEO%pY3c1wx3&97Hy#^$cW-y6tA|xQ z=%@6k2rNGcE7K-$nYlpk?{0T`-TkJqryF}sqxnGZ=||w%&I*AI12Q?c$eAj;nQa<5wj%({r=eU!UHUYu<5~yEEc! e?7)}uC5t-F(d)M_W+x9H8GfVd@ z#MW{lxp3eQAP&hf;ve;uQ*PXlED(Iv``)Y_jM7$@yQ+G+tGb>WR;wig3(Q(GYSrDpB zx0tzZGmF`f)L{-JE2v}l`2GA=|D-re*YfVh-jzGClq3x#%%T`qFrQ^nw+$7-9J?TN zn-D)v{koRd-;5JpkLcFnpmhOr1yd#fJ+pJtJ^6hiH9{HR2RC z#{tLF-z-Q=Z{em`u*6#{~bj%Vi$G)^V7`yIiQ3 z`&O1bV0=v8d-I^9R}vnIl`z`od*=_@H?SYyyYydneI4i9`p{;rs>2syb9N zAfHl&)pjS}QAeR+UyFO$S}x!i>g24_ceM*ku1YsW)*kFyn%$*JV~ted(=d;=l{fm- z8>*qonx$oA>gbR>k{d7VNS~)_3$Q89gP=@Kd=bEK9n&&h)BW2k9JJbfZQ4yJueX5!of0JbU&Yc_G8%f7OCh081|xBGIuY`f1~!dc$)x)M;~anSq6F;b5j0u`^Y~I*%u=D`wf2>(5QCb@KpMnnefAb z9^WH5W!?Oc`W%NYS7!7%yN^D@+h2$_m!GfAAyuJz{mG0f7oVfbC^I%|o;ZV##aRMA zfpY}NUj-Rm&`SxrW5Ol0b7;SK9gGW@l3FGY>BKA2pCI~FaUbR#9p^uSn;R{&XSA%` zros;Bwu;RB*&fin5#5=hFMwXs`^!`G3h32IIp zHem*`h|@%2cxhOAC#$Sl9u;2{fI&sBIrE-n_tmXITlH+IuPuo;p{hDFQ8NSNoS?1K zQ-HpKC0z86tzxHGByg0#5`iNG#%6el(3c5lgPg!hot-{Q23njr8Xm4rhK!Afbot;` zS#u~C>Rv=BmsaRHmY)?*sv|RJPPhVjfjUoD%y+pU�-v(YLTuz6W4fj_K%%=%6AV zu=>M24*ox=Q6J!|h(4Vh3qY=6%Fh5MYZP}_Rd?Sg80!UKi@NtF%msEt_uhmRfi3C& zn=p^raEha9aat2yv$V#|o00xyfa_?`nhc~}`W-CDbst$h^sVSxC)~Pc_3U1u=k(lO zvFG&`9#?UX)f7|<5P#Z5D*S#2o~AqIL{z{LV~5agpdEr-<4H--ZOY9z>% literal 0 HcmV?d00001 diff --git a/server/bloom_inference/pb/generate_pb2.py b/server/bloom_inference/pb/generate_pb2.py new file mode 100644 index 000000000..e55a2c77b --- /dev/null +++ b/server/bloom_inference/pb/generate_pb2.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: generate.proto +"""Generated protocol buffer code.""" +from google.protobuf.internal import builder as _builder +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0egenerate.proto\x12\x0bgenerate.v1\"(\n\x18ServiceDiscoveryResponse\x12\x0c\n\x04urls\x18\x01 \x03(\t\"^\n\x16LogitsWarperParameters\x12\x13\n\x0btemperature\x18\x01 \x01(\x02\x12\r\n\x05top_k\x18\x02 \x01(\r\x12\r\n\x05top_p\x18\x03 \x01(\x02\x12\x11\n\tdo_sample\x18\x04 \x01(\x08\"v\n\x07Request\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x0e\n\x06inputs\x18\x02 \x01(\t\x12\x37\n\nparameters\x18\x03 \x01(\x0b\x32#.generate.v1.LogitsWarperParameters\x12\x16\n\x0emax_new_tokens\x18\x04 \x01(\r\";\n\x05\x42\x61tch\x12\n\n\x02id\x18\x01 \x01(\x04\x12&\n\x08requests\x18\x02 \x03(\x0b\x32\x14.generate.v1.Request\"\x7f\n\x0b\x42\x61tchCached\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x13\n\x0brequest_ids\x18\x02 \x03(\x04\x12\x18\n\x10\x62\x61tch_cached_ids\x18\x03 \x03(\x04\x12\x18\n\x10total_batch_size\x18\x04 \x01(\r\x12\x1b\n\x13max_sequence_length\x18\x05 \x01(\r\"0\n\x12\x46inishedGeneration\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x0e\n\x06output\x18\x02 \x01(\t\"F\n\nCacheEntry\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x13\n\x0brequest_ids\x18\x02 \x03(\x04\x12\x17\n\x0fsequence_length\x18\x03 \x01(\r\"\x80\x01\n\x08Response\x12\x31\n\x08\x66inished\x18\x01 \x03(\x0b\x32\x1f.generate.v1.FinishedGeneration\x12\x31\n\x0b\x63\x61\x63he_entry\x18\x02 \x01(\x0b\x32\x17.generate.v1.CacheEntryH\x00\x88\x01\x01\x42\x0e\n\x0c_cache_entry\"\x07\n\x05\x45mpty2\x94\x02\n\x0eTextGeneration\x12O\n\x10ServiceDiscovery\x12\x12.generate.v1.Empty\x1a%.generate.v1.ServiceDiscoveryResponse\"\x00\x12\x34\n\nClearCache\x12\x12.generate.v1.Empty\x1a\x12.generate.v1.Empty\x12\x35\n\x08Generate\x12\x12.generate.v1.Batch\x1a\x15.generate.v1.Response\x12\x44\n\x11GenerateWithCache\x12\x18.generate.v1.BatchCached\x1a\x15.generate.v1.Responseb\x06proto3') + +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'generate_pb2', globals()) +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + _SERVICEDISCOVERYRESPONSE._serialized_start=31 + _SERVICEDISCOVERYRESPONSE._serialized_end=71 + _LOGITSWARPERPARAMETERS._serialized_start=73 + _LOGITSWARPERPARAMETERS._serialized_end=167 + _REQUEST._serialized_start=169 + _REQUEST._serialized_end=287 + _BATCH._serialized_start=289 + _BATCH._serialized_end=348 + _BATCHCACHED._serialized_start=350 + _BATCHCACHED._serialized_end=477 + _FINISHEDGENERATION._serialized_start=479 + _FINISHEDGENERATION._serialized_end=527 + _CACHEENTRY._serialized_start=529 + _CACHEENTRY._serialized_end=599 + _RESPONSE._serialized_start=602 + _RESPONSE._serialized_end=730 + _EMPTY._serialized_start=732 + _EMPTY._serialized_end=739 + _TEXTGENERATION._serialized_start=742 + _TEXTGENERATION._serialized_end=1018 +# @@protoc_insertion_point(module_scope) diff --git a/server/bloom_inference/pb/generate_pb2.py-e b/server/bloom_inference/pb/generate_pb2.py-e new file mode 100644 index 000000000..e55a2c77b --- /dev/null +++ b/server/bloom_inference/pb/generate_pb2.py-e @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: generate.proto +"""Generated protocol buffer code.""" +from google.protobuf.internal import builder as _builder +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0egenerate.proto\x12\x0bgenerate.v1\"(\n\x18ServiceDiscoveryResponse\x12\x0c\n\x04urls\x18\x01 \x03(\t\"^\n\x16LogitsWarperParameters\x12\x13\n\x0btemperature\x18\x01 \x01(\x02\x12\r\n\x05top_k\x18\x02 \x01(\r\x12\r\n\x05top_p\x18\x03 \x01(\x02\x12\x11\n\tdo_sample\x18\x04 \x01(\x08\"v\n\x07Request\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x0e\n\x06inputs\x18\x02 \x01(\t\x12\x37\n\nparameters\x18\x03 \x01(\x0b\x32#.generate.v1.LogitsWarperParameters\x12\x16\n\x0emax_new_tokens\x18\x04 \x01(\r\";\n\x05\x42\x61tch\x12\n\n\x02id\x18\x01 \x01(\x04\x12&\n\x08requests\x18\x02 \x03(\x0b\x32\x14.generate.v1.Request\"\x7f\n\x0b\x42\x61tchCached\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x13\n\x0brequest_ids\x18\x02 \x03(\x04\x12\x18\n\x10\x62\x61tch_cached_ids\x18\x03 \x03(\x04\x12\x18\n\x10total_batch_size\x18\x04 \x01(\r\x12\x1b\n\x13max_sequence_length\x18\x05 \x01(\r\"0\n\x12\x46inishedGeneration\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x0e\n\x06output\x18\x02 \x01(\t\"F\n\nCacheEntry\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x13\n\x0brequest_ids\x18\x02 \x03(\x04\x12\x17\n\x0fsequence_length\x18\x03 \x01(\r\"\x80\x01\n\x08Response\x12\x31\n\x08\x66inished\x18\x01 \x03(\x0b\x32\x1f.generate.v1.FinishedGeneration\x12\x31\n\x0b\x63\x61\x63he_entry\x18\x02 \x01(\x0b\x32\x17.generate.v1.CacheEntryH\x00\x88\x01\x01\x42\x0e\n\x0c_cache_entry\"\x07\n\x05\x45mpty2\x94\x02\n\x0eTextGeneration\x12O\n\x10ServiceDiscovery\x12\x12.generate.v1.Empty\x1a%.generate.v1.ServiceDiscoveryResponse\"\x00\x12\x34\n\nClearCache\x12\x12.generate.v1.Empty\x1a\x12.generate.v1.Empty\x12\x35\n\x08Generate\x12\x12.generate.v1.Batch\x1a\x15.generate.v1.Response\x12\x44\n\x11GenerateWithCache\x12\x18.generate.v1.BatchCached\x1a\x15.generate.v1.Responseb\x06proto3') + +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'generate_pb2', globals()) +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + _SERVICEDISCOVERYRESPONSE._serialized_start=31 + _SERVICEDISCOVERYRESPONSE._serialized_end=71 + _LOGITSWARPERPARAMETERS._serialized_start=73 + _LOGITSWARPERPARAMETERS._serialized_end=167 + _REQUEST._serialized_start=169 + _REQUEST._serialized_end=287 + _BATCH._serialized_start=289 + _BATCH._serialized_end=348 + _BATCHCACHED._serialized_start=350 + _BATCHCACHED._serialized_end=477 + _FINISHEDGENERATION._serialized_start=479 + _FINISHEDGENERATION._serialized_end=527 + _CACHEENTRY._serialized_start=529 + _CACHEENTRY._serialized_end=599 + _RESPONSE._serialized_start=602 + _RESPONSE._serialized_end=730 + _EMPTY._serialized_start=732 + _EMPTY._serialized_end=739 + _TEXTGENERATION._serialized_start=742 + _TEXTGENERATION._serialized_end=1018 +# @@protoc_insertion_point(module_scope) diff --git a/server/bloom_inference/pb/generate_pb2_grpc.py b/server/bloom_inference/pb/generate_pb2_grpc.py new file mode 100644 index 000000000..e8eb63f0c --- /dev/null +++ b/server/bloom_inference/pb/generate_pb2_grpc.py @@ -0,0 +1,169 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + +from . import generate_pb2 as generate__pb2 + + +class TextGenerationStub(object): + """Missing associated documentation comment in .proto file.""" + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.ServiceDiscovery = channel.unary_unary( + '/generate.v1.TextGeneration/ServiceDiscovery', + request_serializer=generate__pb2.Empty.SerializeToString, + response_deserializer=generate__pb2.ServiceDiscoveryResponse.FromString, + ) + self.ClearCache = channel.unary_unary( + '/generate.v1.TextGeneration/ClearCache', + request_serializer=generate__pb2.Empty.SerializeToString, + response_deserializer=generate__pb2.Empty.FromString, + ) + self.Generate = channel.unary_unary( + '/generate.v1.TextGeneration/Generate', + request_serializer=generate__pb2.Batch.SerializeToString, + response_deserializer=generate__pb2.Response.FromString, + ) + self.GenerateWithCache = channel.unary_unary( + '/generate.v1.TextGeneration/GenerateWithCache', + request_serializer=generate__pb2.BatchCached.SerializeToString, + response_deserializer=generate__pb2.Response.FromString, + ) + + +class TextGenerationServicer(object): + """Missing associated documentation comment in .proto file.""" + + def ServiceDiscovery(self, request, context): + """/ Service discovery + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def ClearCache(self, request, context): + """/ Empties batch cache + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Generate(self, request, context): + """/ Generate tokens for a batch without cache + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GenerateWithCache(self, request, context): + """/ Generate tokens for a batch with cache + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_TextGenerationServicer_to_server(servicer, server): + rpc_method_handlers = { + 'ServiceDiscovery': grpc.unary_unary_rpc_method_handler( + servicer.ServiceDiscovery, + request_deserializer=generate__pb2.Empty.FromString, + response_serializer=generate__pb2.ServiceDiscoveryResponse.SerializeToString, + ), + 'ClearCache': grpc.unary_unary_rpc_method_handler( + servicer.ClearCache, + request_deserializer=generate__pb2.Empty.FromString, + response_serializer=generate__pb2.Empty.SerializeToString, + ), + 'Generate': grpc.unary_unary_rpc_method_handler( + servicer.Generate, + request_deserializer=generate__pb2.Batch.FromString, + response_serializer=generate__pb2.Response.SerializeToString, + ), + 'GenerateWithCache': grpc.unary_unary_rpc_method_handler( + servicer.GenerateWithCache, + request_deserializer=generate__pb2.BatchCached.FromString, + response_serializer=generate__pb2.Response.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'generate.v1.TextGeneration', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + + + # This class is part of an EXPERIMENTAL API. +class TextGeneration(object): + """Missing associated documentation comment in .proto file.""" + + @staticmethod + def ServiceDiscovery(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/generate.v1.TextGeneration/ServiceDiscovery', + generate__pb2.Empty.SerializeToString, + generate__pb2.ServiceDiscoveryResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def ClearCache(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/generate.v1.TextGeneration/ClearCache', + generate__pb2.Empty.SerializeToString, + generate__pb2.Empty.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Generate(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/generate.v1.TextGeneration/Generate', + generate__pb2.Batch.SerializeToString, + generate__pb2.Response.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def GenerateWithCache(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/generate.v1.TextGeneration/GenerateWithCache', + generate__pb2.BatchCached.SerializeToString, + generate__pb2.Response.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/server/bloom_inference/pb/generate_pb2_grpc.py-e b/server/bloom_inference/pb/generate_pb2_grpc.py-e new file mode 100644 index 000000000..541a6b8cb --- /dev/null +++ b/server/bloom_inference/pb/generate_pb2_grpc.py-e @@ -0,0 +1,169 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + +import generate_pb2 as generate__pb2 + + +class TextGenerationStub(object): + """Missing associated documentation comment in .proto file.""" + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.ServiceDiscovery = channel.unary_unary( + '/generate.v1.TextGeneration/ServiceDiscovery', + request_serializer=generate__pb2.Empty.SerializeToString, + response_deserializer=generate__pb2.ServiceDiscoveryResponse.FromString, + ) + self.ClearCache = channel.unary_unary( + '/generate.v1.TextGeneration/ClearCache', + request_serializer=generate__pb2.Empty.SerializeToString, + response_deserializer=generate__pb2.Empty.FromString, + ) + self.Generate = channel.unary_unary( + '/generate.v1.TextGeneration/Generate', + request_serializer=generate__pb2.Batch.SerializeToString, + response_deserializer=generate__pb2.Response.FromString, + ) + self.GenerateWithCache = channel.unary_unary( + '/generate.v1.TextGeneration/GenerateWithCache', + request_serializer=generate__pb2.BatchCached.SerializeToString, + response_deserializer=generate__pb2.Response.FromString, + ) + + +class TextGenerationServicer(object): + """Missing associated documentation comment in .proto file.""" + + def ServiceDiscovery(self, request, context): + """/ Service discovery + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def ClearCache(self, request, context): + """/ Empties batch cache + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Generate(self, request, context): + """/ Generate tokens for a batch without cache + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GenerateWithCache(self, request, context): + """/ Generate tokens for a batch with cache + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_TextGenerationServicer_to_server(servicer, server): + rpc_method_handlers = { + 'ServiceDiscovery': grpc.unary_unary_rpc_method_handler( + servicer.ServiceDiscovery, + request_deserializer=generate__pb2.Empty.FromString, + response_serializer=generate__pb2.ServiceDiscoveryResponse.SerializeToString, + ), + 'ClearCache': grpc.unary_unary_rpc_method_handler( + servicer.ClearCache, + request_deserializer=generate__pb2.Empty.FromString, + response_serializer=generate__pb2.Empty.SerializeToString, + ), + 'Generate': grpc.unary_unary_rpc_method_handler( + servicer.Generate, + request_deserializer=generate__pb2.Batch.FromString, + response_serializer=generate__pb2.Response.SerializeToString, + ), + 'GenerateWithCache': grpc.unary_unary_rpc_method_handler( + servicer.GenerateWithCache, + request_deserializer=generate__pb2.BatchCached.FromString, + response_serializer=generate__pb2.Response.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'generate.v1.TextGeneration', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + + + # This class is part of an EXPERIMENTAL API. +class TextGeneration(object): + """Missing associated documentation comment in .proto file.""" + + @staticmethod + def ServiceDiscovery(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/generate.v1.TextGeneration/ServiceDiscovery', + generate__pb2.Empty.SerializeToString, + generate__pb2.ServiceDiscoveryResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def ClearCache(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/generate.v1.TextGeneration/ClearCache', + generate__pb2.Empty.SerializeToString, + generate__pb2.Empty.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Generate(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/generate.v1.TextGeneration/Generate', + generate__pb2.Batch.SerializeToString, + generate__pb2.Response.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def GenerateWithCache(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/generate.v1.TextGeneration/GenerateWithCache', + generate__pb2.BatchCached.SerializeToString, + generate__pb2.Response.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/server/bloom_inference/prepare_weights.py b/server/bloom_inference/prepare_weights.py new file mode 100644 index 000000000..6fd4ec50b --- /dev/null +++ b/server/bloom_inference/prepare_weights.py @@ -0,0 +1,124 @@ +import torch + +from pathlib import Path +from tqdm import tqdm + +MODEL_NAME = "bigscience/bloom" + + +def match_suffix(text, suffix): + return text[-len(suffix) :] == suffix + + +def prepare_weights(hub_path: Path, save_path: Path, tp_world_size: int): + save_paths = [ + save_path / f"{MODEL_NAME}_tp-rank-{tp_rank}-of-{tp_world_size}.pty" + for tp_rank in range(tp_world_size) + ] + + if all(save_path.exists() for save_path in save_paths): + print("Weights are already prepared") + return + + shards_state_dicts = [{} for _ in range(tp_world_size)] + + for weight_path in tqdm(hub_path.glob("*.bin")): + state_dict = torch.load(weight_path, map_location="cpu") + + keys = list(state_dict.keys()) + for state_name in keys: + state = state_dict[state_name] + if any( + match_suffix(state_name, candidate) + for candidate in [ + "self_attention.query_key_value.weight", + "self_attention.query_key_value.bias", + "mlp.dense_h_to_4h.weight", + "mlp.dense_h_to_4h.bias", + "word_embeddings.weight", + "lm_head.weight", + ] + ): + output_size = state.shape[0] + assert output_size % tp_world_size == 0 + block_size = output_size // tp_world_size + sharded_weights = torch.split(state, block_size, dim=0) + assert len(sharded_weights) == tp_world_size + for tp_rank, shard in enumerate(sharded_weights): + assert shard.shape[0] == block_size + if match_suffix(state_name, "lm_head.weight"): + shards_state_dicts[tp_rank][state_name] = shard.detach().clone() + else: + shards_state_dicts[tp_rank][ + "transformer." + state_name + ] = shard.detach().clone() + elif any( + match_suffix(state_name, candidate) + for candidate in [ + "self_attention.dense.weight", + "mlp.dense_4h_to_h.weight", + "lm_head.weight", + ] + ): + input_size = state.shape[1] + assert input_size % tp_world_size == 0 + block_size = input_size // tp_world_size + sharded_weights = torch.split(state, block_size, dim=1) + assert len(sharded_weights) == tp_world_size + for tp_rank, shard in enumerate(sharded_weights): + assert shard.shape[1] == block_size + if match_suffix(state_name, "lm_head.weight"): + shards_state_dicts[tp_rank][state_name] = shard.detach().clone() + else: + shards_state_dicts[tp_rank][ + "transformer." + state_name + ] = shard.detach().clone() + elif any( + match_suffix(state_name, candidate) + for candidate in [ + "self_attention.dense.bias", + "mlp.dense_4h_to_h.bias", + ] + ): + shards_state_dicts[0][ + "transformer." + state_name + ] = state.detach().clone() + for tp_rank in range(1, tp_world_size): + shards_state_dicts[tp_rank][ + "transformer." + state_name + ] = torch.zeros_like(state) + else: + # We duplicate parameters across tp ranks + for tp_rank in range(tp_world_size): + shards_state_dicts[tp_rank][ + "transformer." + state_name + ] = state.detach().clone() + + del state_dict[state_name] # delete key from state_dict + del state # delete tensor + + # we save state_dict + for tp_rank, (save_path, shard_state_dict) in enumerate( + zip(save_paths, shards_state_dicts) + ): + save_paths.append(save_path) + save_path.parent.mkdir(parents=True, exist_ok=True) + if save_path.exists(): + print(f"Skipping {save_path} as it already exists") + else: + torch.save(shard_state_dict, save_path) + + return save_paths + + +if __name__ == "__main__": + from argparse import ArgumentParser + + parser = ArgumentParser() + + parser.add_argument("--hub-path", required=True, type=str) + parser.add_argument("--save-path", required=True, type=str) + parser.add_argument("--world-size", required=True, type=int) + args = parser.parse_args() + + prepare_weights(Path(args.hub_path), Path(args.save_path), args.world_size) diff --git a/server/bloom_inference/server.py b/server/bloom_inference/server.py new file mode 100644 index 000000000..c652468ef --- /dev/null +++ b/server/bloom_inference/server.py @@ -0,0 +1,91 @@ +import asyncio +from grpc import aio + +from grpc_reflection.v1alpha import reflection +from pathlib import Path +from typing import Optional, List + +from bloom_inference.cache import Cache +from bloom_inference.model import BLOOM, Batch, BLOOMSharded +from bloom_inference.pb import generate_pb2_grpc, generate_pb2 + + +class TextGeneration(generate_pb2_grpc.TextGenerationServicer): + def __init__(self, model: BLOOM, cache: Cache, server_urls: List[str]): + self.cache = cache + self.model = model + self.server_urls = server_urls + + async def ServiceDiscovery(self, request, context): + return generate_pb2.ServiceDiscoveryResponse(urls=self.server_urls) + + async def ClearCache(self, request, context): + self.cache.clear() + return generate_pb2.Empty() + + async def Generate(self, request, context): + batch = Batch.from_batch_pb(request, self.model.tokenizer, self.model.device) + finished_generations, cache_entry = self.model.generate_token(batch) + self.cache.set(cache_entry) + + return generate_pb2.Response( + finished=[ + finished_generation.to_pb() + for finished_generation in finished_generations + ], + cache_entry=cache_entry.to_pb() if cache_entry else None, + ) + + async def GenerateWithCache(self, request, context): + batch = Batch.from_batch_cached_pb(request, self.cache) + finished_generations, cache_entry = self.model.generate_token(batch) + self.cache.set(cache_entry) + + return generate_pb2.Response( + finished=[ + finished_generation.to_pb() + for finished_generation in finished_generations + ], + cache_entry=cache_entry.to_pb() if cache_entry else None, + ) + + +def serve(model_name, sharded, shard_directory): + async def serve_inner( + model_name: str, + sharded: bool = False, + shard_directory: Optional[Path] = None, + ): + unix_socket_template = "unix:///tmp/bloom-inference-{}" + if sharded: + if shard_directory is None: + raise ValueError("shard_directory must be set when sharded is True") + model = BLOOMSharded(model_name, shard_directory) + server_urls = [ + unix_socket_template.format(rank) for rank in range(model.world_size) + ] + local_url = unix_socket_template.format(model.rank) + else: + model = BLOOM(model_name) + local_url = unix_socket_template.format(0) + server_urls = [local_url] + + server = aio.server() + generate_pb2_grpc.add_TextGenerationServicer_to_server( + TextGeneration(model, Cache(), server_urls), server + ) + SERVICE_NAMES = ( + generate_pb2.DESCRIPTOR.services_by_name["TextGeneration"].full_name, + reflection.SERVICE_NAME, + ) + reflection.enable_server_reflection(SERVICE_NAMES, server) + server.add_insecure_port(local_url) + await server.start() + print("Server started at {}".format(local_url)) + await server.wait_for_termination() + + asyncio.run(serve_inner(model_name, sharded, shard_directory)) + + +if __name__ == "__main__": + serve("bigscience/bloom-560m", True, Path("/tmp/models")) diff --git a/server/bloom_inference/shard_model.py b/server/bloom_inference/shard_model.py new file mode 100644 index 000000000..360e89625 --- /dev/null +++ b/server/bloom_inference/shard_model.py @@ -0,0 +1,102 @@ +from pathlib import Path + +import torch +from torch import nn +from transformers import AutoModelForCausalLM + + +def match_suffix(text, suffix): + return text[-len(suffix) :] == suffix + + +def shard_model(model_name: str, path: Path, tp_world_size: int, dtype: torch.dtype): + """BLOOM specific sharding mechanism""" + save_paths = [ + path / f"{model_name}_tp-rank-{tp_rank}-of-{tp_world_size}.pty" + for tp_rank in range(tp_world_size) + ] + if all(save_path.exists() for save_path in save_paths): + print("Loading already cached values") + return save_paths + + model: nn.Module = AutoModelForCausalLM.from_pretrained( + model_name, torch_dtype=dtype, local_files_only=True + ) + + shards_state_dicts = [{} for _ in range(tp_world_size)] + state_dict = model.state_dict() + keys = list(state_dict.keys()) + for state_name in keys: + print(state_name) + state = state_dict[state_name] + if any( + match_suffix(state_name, candidate) + for candidate in [ + "self_attention.query_key_value.weight", + "self_attention.query_key_value.bias", + "mlp.dense_h_to_4h.weight", + "mlp.dense_h_to_4h.bias", + "transformer.word_embeddings.weight", + "lm_head.weight", + ] + ): + output_size = state.shape[0] + assert output_size % tp_world_size == 0 + block_size = output_size // tp_world_size + sharded_weights = torch.split(state, block_size, dim=0) + assert len(sharded_weights) == tp_world_size + for tp_rank, shard in enumerate(sharded_weights): + assert shard.shape[0] == block_size + shards_state_dicts[tp_rank][state_name] = shard.detach().clone() + elif any( + match_suffix(state_name, candidate) + for candidate in [ + "self_attention.dense.weight", + "mlp.dense_4h_to_h.weight", + "lm_head.weight", + ] + ): + input_size = state.shape[1] + assert input_size % tp_world_size == 0 + block_size = input_size // tp_world_size + sharded_weights = torch.split(state, block_size, dim=1) + assert len(sharded_weights) == tp_world_size + for tp_rank, shard in enumerate(sharded_weights): + assert shard.shape[1] == block_size + shards_state_dicts[tp_rank][state_name] = shard.detach().clone() + elif any( + match_suffix(state_name, candidate) + for candidate in [ + "self_attention.dense.bias", + "mlp.dense_4h_to_h.bias", + ] + ): + shards_state_dicts[0][state_name] = state.detach().clone() + for tp_rank in range(1, tp_world_size): + shards_state_dicts[tp_rank][state_name] = torch.zeros_like(state) + else: + # We duplicate parameters across tp ranks + for tp_rank in range(tp_world_size): + shards_state_dicts[tp_rank][state_name] = state.detach().clone() + + del state_dict[state_name] # delete key from state_dict + del state # delete tensor + + # we save state_dict + for tp_rank, (save_path, shard_state_dict) in enumerate( + zip(save_paths, shards_state_dicts) + ): + save_path.parent.mkdir(parents=True, exist_ok=True) + torch.save(shard_state_dict, save_path) + save_paths.append(save_path) + + return save_paths + + +if __name__ == "__main__": + model_name = "bigscience/bloom" + save_path = Path("/data/shards") + tp_world_size = 8 + dtype = torch.bfloat16 + + shard_model(model_name, save_path, tp_world_size=tp_world_size, dtype=dtype) diff --git a/server/bloom_inference/utils.py b/server/bloom_inference/utils.py new file mode 100644 index 000000000..db02dadba --- /dev/null +++ b/server/bloom_inference/utils.py @@ -0,0 +1,95 @@ +import os +import contextlib +import torch +import torch.distributed +from transformers.generation_logits_process import ( + LogitsProcessorList, + TemperatureLogitsWarper, + TopPLogitsWarper, + TopKLogitsWarper, +) + + +class Sampling: + def __call__(self, logits): + probs = torch.nn.functional.softmax(logits, dim=-1) + next_tokens = torch.multinomial(probs, num_samples=1).squeeze(1) + return next_tokens + + +class Greedy: + def __call__(self, logits): + return logits.argmax(dim=-1) + + +class NextTokenChooser: + def __init__(self, temperature=1.0, top_k=None, top_p=None, do_sample=False): + warpers = LogitsProcessorList() + # the following idea is largely copied from this PR: https://github.com/huggingface/transformers/pull/5420/files + # all samplers can be found in `generation_utils_samplers.py` + sampling = do_sample + if temperature is not None and temperature != 1.0: + temperature = float(temperature) + warpers.append(TemperatureLogitsWarper(temperature)) + sampling = True + if top_k is not None and top_k != 0: + warpers.append(TopKLogitsWarper(top_k=top_k)) + sampling = True + if top_p is not None and top_p < 1.0: + warpers.append(TopPLogitsWarper(top_p=top_p)) + sampling = True + + self.warpers = warpers + self.choice = Sampling() if sampling else Greedy() + + def __call__(self, input_ids, scores): + scores = self.warpers(input_ids, scores) + next_ids = self.choice(scores) + return next_ids.unsqueeze(-1) + + +class StoppingCriteria: + def __init__(self, max_new_tokens=20): + self.max_new_tokens = max_new_tokens + self.current_tokens = 0 + + def __call__(self, all_ids): + self.current_tokens += 1 + if self.current_tokens >= self.max_new_tokens: + return True + return False + + +def initialize_torch_distributed(): + rank = int(os.getenv("RANK", "0")) + world_size = int(os.getenv("WORLD_SIZE", "1")) + + if torch.cuda.is_available(): + # initialized `torch.distributed` + # Set the device id. + assert world_size <= torch.cuda.device_count(), "Each process is one gpu" + device = rank % torch.cuda.device_count() + torch.cuda.set_device(device) + backend = "nccl" + else: + backend = "gloo" + + # Call the init process. + torch.distributed.init_process_group( + backend=backend, + world_size=world_size, + rank=rank, + init_method="tcp://localhost:6000", + ) + + return torch.distributed.distributed_c10d._get_default_group(), rank, world_size + + +@contextlib.contextmanager +def set_default_dtype(dtype): + saved_dtype = torch.get_default_dtype() + torch.set_default_dtype(dtype) + try: + yield + finally: + torch.set_default_dtype(saved_dtype) diff --git a/server/poetry.lock b/server/poetry.lock new file mode 100644 index 000000000..ea20ef269 --- /dev/null +++ b/server/poetry.lock @@ -0,0 +1,480 @@ +[[package]] +name = "accelerate" +version = "0.12.0" +description = "Accelerate" +category = "main" +optional = false +python-versions = ">=3.7.0" + +[package.dependencies] +numpy = ">=1.17" +packaging = ">=20.0" +psutil = "*" +pyyaml = "*" +torch = ">=1.4.0" + +[package.extras] +dev = ["black (>=22.0,<23.0)", "datasets", "deepspeed (<0.7.0)", "evaluate", "flake8 (>=3.8.3)", "hf-doc-builder (>=0.3.0)", "isort (>=5.5.4)", "parameterized", "pytest", "pytest-subtests", "pytest-xdist", "scipy", "sklearn", "tqdm", "transformers"] +quality = ["black (>=22.0,<23.0)", "flake8 (>=3.8.3)", "hf-doc-builder (>=0.3.0)", "isort (>=5.5.4)"] +sagemaker = ["sagemaker"] +test_dev = ["datasets", "deepspeed (<0.7.0)", "evaluate", "scipy", "sklearn", "tqdm", "transformers"] +test_prod = ["parameterized", "pytest", "pytest-subtests", "pytest-xdist"] +test_trackers = ["comet-ml", "tensorboard", "wandb"] +testing = ["datasets", "deepspeed (<0.7.0)", "evaluate", "parameterized", "pytest", "pytest-subtests", "pytest-xdist", "scipy", "sklearn", "tqdm", "transformers"] + +[[package]] +name = "click" +version = "8.1.3" +description = "Composable command line interface toolkit" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "colorama" +version = "0.4.5" +description = "Cross-platform colored terminal text." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[[package]] +name = "grpcio" +version = "1.49.1" +description = "HTTP/2-based RPC framework" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +six = ">=1.5.2" + +[package.extras] +protobuf = ["grpcio-tools (>=1.49.1)"] + +[[package]] +name = "grpcio-reflection" +version = "1.49.1" +description = "Standard Protobuf Reflection Service for gRPC" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +grpcio = ">=1.49.1" +protobuf = ">=4.21.3" + +[[package]] +name = "grpcio-tools" +version = "1.49.1" +description = "Protobuf code generator for gRPC" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +grpcio = ">=1.49.1" +protobuf = ">=4.21.3,<5.0dev" +setuptools = "*" + +[[package]] +name = "numpy" +version = "1.23.3" +description = "NumPy is the fundamental package for array computing with Python." +category = "main" +optional = false +python-versions = ">=3.8" + +[[package]] +name = "packaging" +version = "21.3" +description = "Core utilities for Python packages" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" + +[[package]] +name = "protobuf" +version = "4.21.7" +description = "" +category = "main" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "psutil" +version = "5.9.2" +description = "Cross-platform lib for process and system monitoring in Python." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.extras] +test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] + +[[package]] +name = "pyparsing" +version = "3.0.9" +description = "pyparsing module - Classes and methods to define and execute parsing grammars" +category = "main" +optional = false +python-versions = ">=3.6.8" + +[package.extras] +diagrams = ["jinja2", "railroad-diagrams"] + +[[package]] +name = "PyYAML" +version = "6.0" +description = "YAML parser and emitter for Python" +category = "main" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "setuptools" +version = "65.4.1" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mock", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" + +[[package]] +name = "torch" +version = "1.12.1" +description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" +category = "main" +optional = false +python-versions = ">=3.7.0" + +[package.dependencies] +typing-extensions = "*" + +[[package]] +name = "typer" +version = "0.6.1" +description = "Typer, build great CLIs. Easy to code. Based on Python type hints." +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +click = ">=7.1.1,<9.0.0" + +[package.extras] +all = ["colorama (>=0.4.3,<0.5.0)", "rich (>=10.11.0,<13.0.0)", "shellingham (>=1.3.0,<2.0.0)"] +dev = ["autoflake (>=1.3.1,<2.0.0)", "flake8 (>=3.8.3,<4.0.0)", "pre-commit (>=2.17.0,<3.0.0)"] +doc = ["mdx-include (>=1.4.1,<2.0.0)", "mkdocs (>=1.1.2,<2.0.0)", "mkdocs-material (>=8.1.4,<9.0.0)"] +test = ["black (>=22.3.0,<23.0.0)", "coverage (>=5.2,<6.0)", "isort (>=5.0.6,<6.0.0)", "mypy (==0.910)", "pytest (>=4.4.0,<5.4.0)", "pytest-cov (>=2.10.0,<3.0.0)", "pytest-sugar (>=0.9.4,<0.10.0)", "pytest-xdist (>=1.32.0,<2.0.0)", "rich (>=10.11.0,<13.0.0)", "shellingham (>=1.3.0,<2.0.0)"] + +[[package]] +name = "typing-extensions" +version = "4.3.0" +description = "Backported and Experimental Type Hints for Python 3.7+" +category = "main" +optional = false +python-versions = ">=3.7" + +[metadata] +lock-version = "1.1" +python-versions = "^3.9" +content-hash = "cedd0aebeb3731e2bbddf017a2ee6074c285866354272f8dfe930e9606437a25" + +[metadata.files] +accelerate = [ + {file = "accelerate-0.12.0-py3-none-any.whl", hash = "sha256:7742ca5c9f15dd1e0a283305599c196e260af4717a561d1f544aeab27d828af6"}, + {file = "accelerate-0.12.0.tar.gz", hash = "sha256:e8b119c94fac31877620d5f9de311164ec81fa9dc9e175f0d0d4f50fc8d79473"}, +] +click = [ + {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, + {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, +] +colorama = [ + {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"}, + {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, +] +grpcio = [ + {file = "grpcio-1.49.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:fd86040232e805b8e6378b2348c928490ee595b058ce9aaa27ed8e4b0f172b20"}, + {file = "grpcio-1.49.1-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:6fd0c9cede9552bf00f8c5791d257d5bf3790d7057b26c59df08be5e7a1e021d"}, + {file = "grpcio-1.49.1-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:d0d402e158d4e84e49c158cb5204119d55e1baf363ee98d6cb5dce321c3a065d"}, + {file = "grpcio-1.49.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:822ceec743d42a627e64ea266059a62d214c5a3cdfcd0d7fe2b7a8e4e82527c7"}, + {file = "grpcio-1.49.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2106d9c16527f0a85e2eea6e6b91a74fc99579c60dd810d8690843ea02bc0f5f"}, + {file = "grpcio-1.49.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:52dd02b7e7868233c571b49bc38ebd347c3bb1ff8907bb0cb74cb5f00c790afc"}, + {file = "grpcio-1.49.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:120fecba2ec5d14b5a15d11063b39783fda8dc8d24addd83196acb6582cabd9b"}, + {file = "grpcio-1.49.1-cp310-cp310-win32.whl", hash = "sha256:f1a3b88e3c53c1a6e6bed635ec1bbb92201bb6a1f2db186179f7f3f244829788"}, + {file = "grpcio-1.49.1-cp310-cp310-win_amd64.whl", hash = "sha256:a7d0017b92d3850abea87c1bdec6ea41104e71c77bca44c3e17f175c6700af62"}, + {file = "grpcio-1.49.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:9fb17ff8c0d56099ac6ebfa84f670c5a62228d6b5c695cf21c02160c2ac1446b"}, + {file = "grpcio-1.49.1-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:075f2d06e3db6b48a2157a1bcd52d6cbdca980dd18988fe6afdb41795d51625f"}, + {file = "grpcio-1.49.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:46d93a1b4572b461a227f1db6b8d35a88952db1c47e5fadcf8b8a2f0e1dd9201"}, + {file = "grpcio-1.49.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc79b2b37d779ac42341ddef40ad5bf0966a64af412c89fc2b062e3ddabb093f"}, + {file = "grpcio-1.49.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:5f8b3a971c7820ea9878f3fd70086240a36aeee15d1b7e9ecbc2743b0e785568"}, + {file = "grpcio-1.49.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:49b301740cf5bc8fed4fee4c877570189ae3951432d79fa8e524b09353659811"}, + {file = "grpcio-1.49.1-cp311-cp311-win32.whl", hash = "sha256:1c66a25afc6c71d357867b341da594a5587db5849b48f4b7d5908d236bb62ede"}, + {file = "grpcio-1.49.1-cp311-cp311-win_amd64.whl", hash = "sha256:6b6c3a95d27846f4145d6967899b3ab25fffc6ae99544415e1adcacef84842d2"}, + {file = "grpcio-1.49.1-cp37-cp37m-linux_armv7l.whl", hash = "sha256:1cc400c8a2173d1c042997d98a9563e12d9bb3fb6ad36b7f355bc77c7663b8af"}, + {file = "grpcio-1.49.1-cp37-cp37m-macosx_10_10_x86_64.whl", hash = "sha256:34f736bd4d0deae90015c0e383885b431444fe6b6c591dea288173df20603146"}, + {file = "grpcio-1.49.1-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:196082b9c89ebf0961dcd77cb114bed8171964c8e3063b9da2fb33536a6938ed"}, + {file = "grpcio-1.49.1-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8c9f89c42749890618cd3c2464e1fbf88446e3d2f67f1e334c8e5db2f3272bbd"}, + {file = "grpcio-1.49.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64419cb8a5b612cdb1550c2fd4acbb7d4fb263556cf4625f25522337e461509e"}, + {file = "grpcio-1.49.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:8a5272061826e6164f96e3255405ef6f73b88fd3e8bef464c7d061af8585ac62"}, + {file = "grpcio-1.49.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ea9d0172445241ad7cb49577314e39d0af2c5267395b3561d7ced5d70458a9f3"}, + {file = "grpcio-1.49.1-cp37-cp37m-win32.whl", hash = "sha256:2070e87d95991473244c72d96d13596c751cb35558e11f5df5414981e7ed2492"}, + {file = "grpcio-1.49.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fcedcab49baaa9db4a2d240ac81f2d57eb0052b1c6a9501b46b8ae912720fbf"}, + {file = "grpcio-1.49.1-cp38-cp38-linux_armv7l.whl", hash = "sha256:afbb3475cf7f4f7d380c2ca37ee826e51974f3e2665613996a91d6a58583a534"}, + {file = "grpcio-1.49.1-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:a4f9ba141380abde6c3adc1727f21529137a2552002243fa87c41a07e528245c"}, + {file = "grpcio-1.49.1-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:cf0a1fb18a7204b9c44623dfbd1465b363236ce70c7a4ed30402f9f60d8b743b"}, + {file = "grpcio-1.49.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:17bb6fe72784b630728c6cff9c9d10ccc3b6d04e85da6e0a7b27fb1d135fac62"}, + {file = "grpcio-1.49.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:18305d5a082d1593b005a895c10041f833b16788e88b02bb81061f5ebcc465df"}, + {file = "grpcio-1.49.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:b6a1b39e59ac5a3067794a0e498911cf2e37e4b19ee9e9977dc5e7051714f13f"}, + {file = "grpcio-1.49.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0e20d59aafc086b1cc68400463bddda6e41d3e5ed30851d1e2e0f6a2e7e342d3"}, + {file = "grpcio-1.49.1-cp38-cp38-win32.whl", hash = "sha256:e1e83233d4680863a421f3ee4a7a9b80d33cd27ee9ed7593bc93f6128302d3f2"}, + {file = "grpcio-1.49.1-cp38-cp38-win_amd64.whl", hash = "sha256:221d42c654d2a41fa31323216279c73ed17d92f533bc140a3390cc1bd78bf63c"}, + {file = "grpcio-1.49.1-cp39-cp39-linux_armv7l.whl", hash = "sha256:fa9e6e61391e99708ac87fc3436f6b7b9c6b845dc4639b406e5e61901e1aacde"}, + {file = "grpcio-1.49.1-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:9b449e966ef518ce9c860d21f8afe0b0f055220d95bc710301752ac1db96dd6a"}, + {file = "grpcio-1.49.1-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:aa34d2ad9f24e47fa9a3172801c676e4037d862247e39030165fe83821a7aafd"}, + {file = "grpcio-1.49.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5207f4eed1b775d264fcfe379d8541e1c43b878f2b63c0698f8f5c56c40f3d68"}, + {file = "grpcio-1.49.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b24a74651438d45619ac67004638856f76cc13d78b7478f2457754cbcb1c8ad"}, + {file = "grpcio-1.49.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:fe763781669790dc8b9618e7e677c839c87eae6cf28b655ee1fa69ae04eea03f"}, + {file = "grpcio-1.49.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2f2ff7ba0f8f431f32d4b4bc3a3713426949d3533b08466c4ff1b2b475932ca8"}, + {file = "grpcio-1.49.1-cp39-cp39-win32.whl", hash = "sha256:08ff74aec8ff457a89b97152d36cb811dcc1d17cd5a92a65933524e363327394"}, + {file = "grpcio-1.49.1-cp39-cp39-win_amd64.whl", hash = "sha256:274ffbb39717918c514b35176510ae9be06e1d93121e84d50b350861dcb9a705"}, + {file = "grpcio-1.49.1.tar.gz", hash = "sha256:d4725fc9ec8e8822906ae26bb26f5546891aa7fbc3443de970cc556d43a5c99f"}, +] +grpcio-reflection = [ + {file = "grpcio-reflection-1.49.1.tar.gz", hash = "sha256:b755dfe61d5255a02fb8d0d845bd0027847dee68bf0763a2b286d664ed07ec4d"}, + {file = "grpcio_reflection-1.49.1-py3-none-any.whl", hash = "sha256:70a325a83c1c1ab583d368711e5733cbef5e068ad2c17cbe77df6e47e0311d1f"}, +] +grpcio-tools = [ + {file = "grpcio-tools-1.49.1.tar.gz", hash = "sha256:84cc64e5b46bad43d5d7bd2fd772b656eba0366961187a847e908e2cb735db91"}, + {file = "grpcio_tools-1.49.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:2dfb6c7ece84d46bd690b23d3e060d18115c8bc5047d2e8a33e6747ed323a348"}, + {file = "grpcio_tools-1.49.1-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:8f452a107c054a04db2570f7851a07f060313c6e841b0d394ce6030d598290e6"}, + {file = "grpcio_tools-1.49.1-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:6a198871b582287213c4d70792bf275e1d7cf34eed1d019f534ddf4cd15ab039"}, + {file = "grpcio_tools-1.49.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a0cca67a7d0287bdc855d81fdd38dc949c4273273a74f832f9e520abe4f20bc6"}, + {file = "grpcio_tools-1.49.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bdaff4c89eecb37c247b93025410db68114d97fa093cbb028e9bd7cda5912473"}, + {file = "grpcio_tools-1.49.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:bb8773118ad315db317d7b22b5ff75d649ca20931733281209e7cbd8c0fad53e"}, + {file = "grpcio_tools-1.49.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7cc5534023735b8a8f56760b7c533918f874ce5a9064d7c5456d2709ae2b31f9"}, + {file = "grpcio_tools-1.49.1-cp310-cp310-win32.whl", hash = "sha256:d277642acbe305f5586f9597b78fb9970d6633eb9f89c61e429c92c296c37129"}, + {file = "grpcio_tools-1.49.1-cp310-cp310-win_amd64.whl", hash = "sha256:eed599cf08fc1a06c72492d3c5750c32f58de3750eddd984af1f257c14326701"}, + {file = "grpcio_tools-1.49.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:9e5c13809ab2f245398e8446c4c3b399a62d591db651e46806cccf52a700452e"}, + {file = "grpcio_tools-1.49.1-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:ab3d0ee9623720ee585fdf3753b3755d3144a4a8ae35bca8e3655fa2f41056be"}, + {file = "grpcio_tools-1.49.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ba87e3512bc91d78bf9febcfb522eadda171d2d4ddaf886066b0f01aa4929ad"}, + {file = "grpcio_tools-1.49.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13e13b3643e7577a3ec13b79689eb4d7548890b1e104c04b9ed6557a3c3dd452"}, + {file = "grpcio_tools-1.49.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:324f67d9cb4b7058b6ce45352fb64c20cc1fa04c34d97ad44772cfe6a4ae0cf5"}, + {file = "grpcio_tools-1.49.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a64bab81b220c50033f584f57978ebbea575f09c1ccee765cd5c462177988098"}, + {file = "grpcio_tools-1.49.1-cp311-cp311-win32.whl", hash = "sha256:f632d376f92f23e5931697a3acf1b38df7eb719774213d93c52e02acd2d529ac"}, + {file = "grpcio_tools-1.49.1-cp311-cp311-win_amd64.whl", hash = "sha256:28ff2b978d9509474928b9c096a0cce4eaa9c8f7046136aee1545f6211ed8126"}, + {file = "grpcio_tools-1.49.1-cp37-cp37m-linux_armv7l.whl", hash = "sha256:46afd3cb7e555187451a5d283f108cdef397952a662cb48680afc615b158864a"}, + {file = "grpcio_tools-1.49.1-cp37-cp37m-macosx_10_10_x86_64.whl", hash = "sha256:9284568b728e41fa8f7e9c2e7399545d605f75d8072ef0e9aa2a05655cb679eb"}, + {file = "grpcio_tools-1.49.1-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:aa34442cf787732cb41f2aa6172007e24f480b8b9d3dc5166de80d63e9072ea4"}, + {file = "grpcio_tools-1.49.1-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3b8c9eb5a4250905414cd53a68caea3eb8f0c515aadb689e6e81b71ebe9ab5c6"}, + {file = "grpcio_tools-1.49.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab15db024051bf21feb21c29cb2c3ea0a2e4f5cf341d46ef76e17fcf6aaef164"}, + {file = "grpcio_tools-1.49.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:502084b622f758bef620a9107c2db9fcdf66d26c7e0e481d6bb87db4dc917d70"}, + {file = "grpcio_tools-1.49.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:4085890b77c640085f82bf1e90a0ea166ce48000bc2f5180914b974783c9c0a8"}, + {file = "grpcio_tools-1.49.1-cp37-cp37m-win32.whl", hash = "sha256:da0edb984699769ce02e18e3392d54b59a7a3f93acd285a68043f5bde4fc028e"}, + {file = "grpcio_tools-1.49.1-cp37-cp37m-win_amd64.whl", hash = "sha256:9887cd622770271101a7dd1832845d64744c3f88fd11ccb2620394079197a42e"}, + {file = "grpcio_tools-1.49.1-cp38-cp38-linux_armv7l.whl", hash = "sha256:8440fe7dae6a40c279e3a24b82793735babd38ecbb0d07bb712ff9c8963185d9"}, + {file = "grpcio_tools-1.49.1-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:b5de2bb7dd6b6231da9b1556ade981513330b740e767f1d902c71ceee0a7d196"}, + {file = "grpcio_tools-1.49.1-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:1e6f06a763aea7836b63d9c117347f2bf7038008ceef72758815c9e09c5fb1fc"}, + {file = "grpcio_tools-1.49.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e31562f90120318c5395aabec0f2f69ad8c14b6676996b7730d9d2eaf9415d57"}, + {file = "grpcio_tools-1.49.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49ef9a4e389a618157a9daa9fafdfeeaef1ece9adda7f50f85db928f24d4b3e8"}, + {file = "grpcio_tools-1.49.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:b384cb8e8d9bcb55ee8f9b064374561c7a1a05d848249581403d36fc7060032f"}, + {file = "grpcio_tools-1.49.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:73732f77943ac3e898879cbb29c27253aa3c47566b8a59780fd24c6a54de1b66"}, + {file = "grpcio_tools-1.49.1-cp38-cp38-win32.whl", hash = "sha256:b594b2745a5ba9e7a76ce561bc5ab40bc65bb44743c505529b1e4f12af29104d"}, + {file = "grpcio_tools-1.49.1-cp38-cp38-win_amd64.whl", hash = "sha256:680fbc88f8709ddcabb88f86749f2d8e429160890cff2c70680880a6970d4eef"}, + {file = "grpcio_tools-1.49.1-cp39-cp39-linux_armv7l.whl", hash = "sha256:e8c3869121860f6767eedb7d24fc54dfd71e737fdfbb26e1334684606f3274fd"}, + {file = "grpcio_tools-1.49.1-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:73e9d7c886ba10e20c97d1dab0ff961ba5800757ae5e31be21b1cda8130c52f8"}, + {file = "grpcio_tools-1.49.1-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:1760de2dd2c4f08de87b039043a4797f3c17193656e7e3eb84e92f0517083c0c"}, + {file = "grpcio_tools-1.49.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cd4b1e216dd04d9245ee8f4e601a1f98c25e6e417ea5cf8d825c50589a8b447e"}, + {file = "grpcio_tools-1.49.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c1c28751ab5955cae563d07677e799233f0fe1c0fc49d9cbd61ff1957e83617f"}, + {file = "grpcio_tools-1.49.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:c24239c3ee9ed16314c14b4e24437b5079ebc344f343f33629a582f8699f583b"}, + {file = "grpcio_tools-1.49.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:892d3dacf1942820f0b7a868a30e6fbcdf5bec08543b682c7274b0101cee632d"}, + {file = "grpcio_tools-1.49.1-cp39-cp39-win32.whl", hash = "sha256:704d21509ec06efc9d034dbe70e7152715aac004941f4f0f553cf3a0aff15bd5"}, + {file = "grpcio_tools-1.49.1-cp39-cp39-win_amd64.whl", hash = "sha256:1efa0c221c719433f441ac0e026fc3c4dbc9a1a08a552ecdc707775e2f2fbbae"}, +] +numpy = [ + {file = "numpy-1.23.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c9f707b5bb73bf277d812ded9896f9512a43edff72712f31667d0a8c2f8e71ee"}, + {file = "numpy-1.23.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ffcf105ecdd9396e05a8e58e81faaaf34d3f9875f137c7372450baa5d77c9a54"}, + {file = "numpy-1.23.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ea3f98a0ffce3f8f57675eb9119f3f4edb81888b6874bc1953f91e0b1d4f440"}, + {file = "numpy-1.23.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:004f0efcb2fe1c0bd6ae1fcfc69cc8b6bf2407e0f18be308612007a0762b4089"}, + {file = "numpy-1.23.3-cp310-cp310-win32.whl", hash = "sha256:98dcbc02e39b1658dc4b4508442a560fe3ca5ca0d989f0df062534e5ca3a5c1a"}, + {file = "numpy-1.23.3-cp310-cp310-win_amd64.whl", hash = "sha256:39a664e3d26ea854211867d20ebcc8023257c1800ae89773cbba9f9e97bae036"}, + {file = "numpy-1.23.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1f27b5322ac4067e67c8f9378b41c746d8feac8bdd0e0ffede5324667b8a075c"}, + {file = "numpy-1.23.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2ad3ec9a748a8943e6eb4358201f7e1c12ede35f510b1a2221b70af4bb64295c"}, + {file = "numpy-1.23.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bdc9febce3e68b697d931941b263c59e0c74e8f18861f4064c1f712562903411"}, + {file = "numpy-1.23.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:301c00cf5e60e08e04d842fc47df641d4a181e651c7135c50dc2762ffe293dbd"}, + {file = "numpy-1.23.3-cp311-cp311-win32.whl", hash = "sha256:7cd1328e5bdf0dee621912f5833648e2daca72e3839ec1d6695e91089625f0b4"}, + {file = "numpy-1.23.3-cp311-cp311-win_amd64.whl", hash = "sha256:8355fc10fd33a5a70981a5b8a0de51d10af3688d7a9e4a34fcc8fa0d7467bb7f"}, + {file = "numpy-1.23.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bc6e8da415f359b578b00bcfb1d08411c96e9a97f9e6c7adada554a0812a6cc6"}, + {file = "numpy-1.23.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:22d43376ee0acd547f3149b9ec12eec2f0ca4a6ab2f61753c5b29bb3e795ac4d"}, + {file = "numpy-1.23.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a64403f634e5ffdcd85e0b12c08f04b3080d3e840aef118721021f9b48fc1460"}, + {file = "numpy-1.23.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efd9d3abe5774404becdb0748178b48a218f1d8c44e0375475732211ea47c67e"}, + {file = "numpy-1.23.3-cp38-cp38-win32.whl", hash = "sha256:f8c02ec3c4c4fcb718fdf89a6c6f709b14949408e8cf2a2be5bfa9c49548fd85"}, + {file = "numpy-1.23.3-cp38-cp38-win_amd64.whl", hash = "sha256:e868b0389c5ccfc092031a861d4e158ea164d8b7fdbb10e3b5689b4fc6498df6"}, + {file = "numpy-1.23.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:09f6b7bdffe57fc61d869a22f506049825d707b288039d30f26a0d0d8ea05164"}, + {file = "numpy-1.23.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8c79d7cf86d049d0c5089231a5bcd31edb03555bd93d81a16870aa98c6cfb79d"}, + {file = "numpy-1.23.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e5d5420053bbb3dd64c30e58f9363d7a9c27444c3648e61460c1237f9ec3fa14"}, + {file = "numpy-1.23.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5422d6a1ea9b15577a9432e26608c73a78faf0b9039437b075cf322c92e98e7"}, + {file = "numpy-1.23.3-cp39-cp39-win32.whl", hash = "sha256:c1ba66c48b19cc9c2975c0d354f24058888cdc674bebadceb3cdc9ec403fb5d1"}, + {file = "numpy-1.23.3-cp39-cp39-win_amd64.whl", hash = "sha256:78a63d2df1d947bd9d1b11d35564c2f9e4b57898aae4626638056ec1a231c40c"}, + {file = "numpy-1.23.3-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:17c0e467ade9bda685d5ac7f5fa729d8d3e76b23195471adae2d6a6941bd2c18"}, + {file = "numpy-1.23.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:91b8d6768a75247026e951dce3b2aac79dc7e78622fc148329135ba189813584"}, + {file = "numpy-1.23.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:94c15ca4e52671a59219146ff584488907b1f9b3fc232622b47e2cf832e94fb8"}, + {file = "numpy-1.23.3.tar.gz", hash = "sha256:51bf49c0cd1d52be0a240aa66f3458afc4b95d8993d2d04f0d91fa60c10af6cd"}, +] +packaging = [ + {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, + {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, +] +protobuf = [ + {file = "protobuf-4.21.7-cp310-abi3-win32.whl", hash = "sha256:c7cb105d69a87416bd9023e64324e1c089593e6dae64d2536f06bcbe49cd97d8"}, + {file = "protobuf-4.21.7-cp310-abi3-win_amd64.whl", hash = "sha256:3ec85328a35a16463c6f419dbce3c0fc42b3e904d966f17f48bae39597c7a543"}, + {file = "protobuf-4.21.7-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:db9056b6a11cb5131036d734bcbf91ef3ef9235d6b681b2fc431cbfe5a7f2e56"}, + {file = "protobuf-4.21.7-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:ca200645d6235ce0df3ccfdff1567acbab35c4db222a97357806e015f85b5744"}, + {file = "protobuf-4.21.7-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:b019c79e23a80735cc8a71b95f76a49a262f579d6b84fd20a0b82279f40e2cc1"}, + {file = "protobuf-4.21.7-cp37-cp37m-win32.whl", hash = "sha256:d3f89ccf7182293feba2de2739c8bf34fed1ed7c65a5cf987be00311acac57c1"}, + {file = "protobuf-4.21.7-cp37-cp37m-win_amd64.whl", hash = "sha256:a74d96cd960b87b4b712797c741bb3ea3a913f5c2dc4b6cbe9c0f8360b75297d"}, + {file = "protobuf-4.21.7-cp38-cp38-win32.whl", hash = "sha256:8e09d1916386eca1ef1353767b6efcebc0a6859ed7f73cb7fb974feba3184830"}, + {file = "protobuf-4.21.7-cp38-cp38-win_amd64.whl", hash = "sha256:9e355f2a839d9930d83971b9f562395e13493f0e9211520f8913bd11efa53c02"}, + {file = "protobuf-4.21.7-cp39-cp39-win32.whl", hash = "sha256:f370c0a71712f8965023dd5b13277444d3cdfecc96b2c778b0e19acbfd60df6e"}, + {file = "protobuf-4.21.7-cp39-cp39-win_amd64.whl", hash = "sha256:9643684232b6b340b5e63bb69c9b4904cdd39e4303d498d1a92abddc7e895b7f"}, + {file = "protobuf-4.21.7-py2.py3-none-any.whl", hash = "sha256:8066322588d4b499869bf9f665ebe448e793036b552f68c585a9b28f1e393f66"}, + {file = "protobuf-4.21.7-py3-none-any.whl", hash = "sha256:58b81358ec6c0b5d50df761460ae2db58405c063fd415e1101209221a0a810e1"}, + {file = "protobuf-4.21.7.tar.gz", hash = "sha256:71d9dba03ed3432c878a801e2ea51e034b0ea01cf3a4344fb60166cb5f6c8757"}, +] +psutil = [ + {file = "psutil-5.9.2-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:8f024fbb26c8daf5d70287bb3edfafa22283c255287cf523c5d81721e8e5d82c"}, + {file = "psutil-5.9.2-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:b2f248ffc346f4f4f0d747ee1947963613216b06688be0be2e393986fe20dbbb"}, + {file = "psutil-5.9.2-cp27-cp27m-win32.whl", hash = "sha256:b1928b9bf478d31fdffdb57101d18f9b70ed4e9b0e41af751851813547b2a9ab"}, + {file = "psutil-5.9.2-cp27-cp27m-win_amd64.whl", hash = "sha256:404f4816c16a2fcc4eaa36d7eb49a66df2d083e829d3e39ee8759a411dbc9ecf"}, + {file = "psutil-5.9.2-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:94e621c6a4ddb2573d4d30cba074f6d1aa0186645917df42c811c473dd22b339"}, + {file = "psutil-5.9.2-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:256098b4f6ffea6441eb54ab3eb64db9ecef18f6a80d7ba91549195d55420f84"}, + {file = "psutil-5.9.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:614337922702e9be37a39954d67fdb9e855981624d8011a9927b8f2d3c9625d9"}, + {file = "psutil-5.9.2-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:39ec06dc6c934fb53df10c1672e299145ce609ff0611b569e75a88f313634969"}, + {file = "psutil-5.9.2-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e3ac2c0375ef498e74b9b4ec56df3c88be43fe56cac465627572dbfb21c4be34"}, + {file = "psutil-5.9.2-cp310-cp310-win32.whl", hash = "sha256:e4c4a7636ffc47b7141864f1c5e7d649f42c54e49da2dd3cceb1c5f5d29bfc85"}, + {file = "psutil-5.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:f4cb67215c10d4657e320037109939b1c1d2fd70ca3d76301992f89fe2edb1f1"}, + {file = "psutil-5.9.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:dc9bda7d5ced744622f157cc8d8bdd51735dafcecff807e928ff26bdb0ff097d"}, + {file = "psutil-5.9.2-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d75291912b945a7351d45df682f9644540d564d62115d4a20d45fa17dc2d48f8"}, + {file = "psutil-5.9.2-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b4018d5f9b6651f9896c7a7c2c9f4652e4eea53f10751c4e7d08a9093ab587ec"}, + {file = "psutil-5.9.2-cp36-cp36m-win32.whl", hash = "sha256:f40ba362fefc11d6bea4403f070078d60053ed422255bd838cd86a40674364c9"}, + {file = "psutil-5.9.2-cp36-cp36m-win_amd64.whl", hash = "sha256:9770c1d25aee91417eba7869139d629d6328a9422ce1cdd112bd56377ca98444"}, + {file = "psutil-5.9.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:42638876b7f5ef43cef8dcf640d3401b27a51ee3fa137cb2aa2e72e188414c32"}, + {file = "psutil-5.9.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:91aa0dac0c64688667b4285fa29354acfb3e834e1fd98b535b9986c883c2ce1d"}, + {file = "psutil-5.9.2-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4fb54941aac044a61db9d8eb56fc5bee207db3bc58645d657249030e15ba3727"}, + {file = "psutil-5.9.2-cp37-cp37m-win32.whl", hash = "sha256:7cbb795dcd8ed8fd238bc9e9f64ab188f3f4096d2e811b5a82da53d164b84c3f"}, + {file = "psutil-5.9.2-cp37-cp37m-win_amd64.whl", hash = "sha256:5d39e3a2d5c40efa977c9a8dd4f679763c43c6c255b1340a56489955dbca767c"}, + {file = "psutil-5.9.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:fd331866628d18223a4265371fd255774affd86244fc307ef66eaf00de0633d5"}, + {file = "psutil-5.9.2-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b315febaebae813326296872fdb4be92ad3ce10d1d742a6b0c49fb619481ed0b"}, + {file = "psutil-5.9.2-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7929a516125f62399d6e8e026129c8835f6c5a3aab88c3fff1a05ee8feb840d"}, + {file = "psutil-5.9.2-cp38-cp38-win32.whl", hash = "sha256:561dec454853846d1dd0247b44c2e66a0a0c490f937086930ec4b8f83bf44f06"}, + {file = "psutil-5.9.2-cp38-cp38-win_amd64.whl", hash = "sha256:67b33f27fc0427483b61563a16c90d9f3b547eeb7af0ef1b9fe024cdc9b3a6ea"}, + {file = "psutil-5.9.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b3591616fa07b15050b2f87e1cdefd06a554382e72866fcc0ab2be9d116486c8"}, + {file = "psutil-5.9.2-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:14b29f581b5edab1f133563272a6011925401804d52d603c5c606936b49c8b97"}, + {file = "psutil-5.9.2-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4642fd93785a29353d6917a23e2ac6177308ef5e8be5cc17008d885cb9f70f12"}, + {file = "psutil-5.9.2-cp39-cp39-win32.whl", hash = "sha256:ed29ea0b9a372c5188cdb2ad39f937900a10fb5478dc077283bf86eeac678ef1"}, + {file = "psutil-5.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:68b35cbff92d1f7103d8f1db77c977e72f49fcefae3d3d2b91c76b0e7aef48b8"}, + {file = "psutil-5.9.2.tar.gz", hash = "sha256:feb861a10b6c3bb00701063b37e4afc754f8217f0f09c42280586bd6ac712b5c"}, +] +pyparsing = [ + {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, + {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, +] +PyYAML = [ + {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, + {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, + {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, + {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, + {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, + {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, + {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4"}, + {file = "PyYAML-6.0-cp36-cp36m-win32.whl", hash = "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293"}, + {file = "PyYAML-6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57"}, + {file = "PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9"}, + {file = "PyYAML-6.0-cp37-cp37m-win32.whl", hash = "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737"}, + {file = "PyYAML-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d"}, + {file = "PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287"}, + {file = "PyYAML-6.0-cp38-cp38-win32.whl", hash = "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78"}, + {file = "PyYAML-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07"}, + {file = "PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b"}, + {file = "PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0"}, + {file = "PyYAML-6.0-cp39-cp39-win32.whl", hash = "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb"}, + {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, + {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, +] +setuptools = [ + {file = "setuptools-65.4.1-py3-none-any.whl", hash = "sha256:1b6bdc6161661409c5f21508763dc63ab20a9ac2f8ba20029aaaa7fdb9118012"}, + {file = "setuptools-65.4.1.tar.gz", hash = "sha256:3050e338e5871e70c72983072fe34f6032ae1cdeeeb67338199c2f74e083a80e"}, +] +six = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] +torch = [ + {file = "torch-1.12.1-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:9c038662db894a23e49e385df13d47b2a777ffd56d9bcd5b832593fab0a7e286"}, + {file = "torch-1.12.1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:4e1b9c14cf13fd2ab8d769529050629a0e68a6fc5cb8e84b4a3cc1dd8c4fe541"}, + {file = "torch-1.12.1-cp310-cp310-win_amd64.whl", hash = "sha256:e9c8f4a311ac29fc7e8e955cfb7733deb5dbe1bdaabf5d4af2765695824b7e0d"}, + {file = "torch-1.12.1-cp310-none-macosx_10_9_x86_64.whl", hash = "sha256:976c3f997cea38ee91a0dd3c3a42322785414748d1761ef926b789dfa97c6134"}, + {file = "torch-1.12.1-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:68104e4715a55c4bb29a85c6a8d57d820e0757da363be1ba680fa8cc5be17b52"}, + {file = "torch-1.12.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:743784ccea0dc8f2a3fe6a536bec8c4763bd82c1352f314937cb4008d4805de1"}, + {file = "torch-1.12.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:b5dbcca369800ce99ba7ae6dee3466607a66958afca3b740690d88168752abcf"}, + {file = "torch-1.12.1-cp37-cp37m-win_amd64.whl", hash = "sha256:f3b52a634e62821e747e872084ab32fbcb01b7fa7dbb7471b6218279f02a178a"}, + {file = "torch-1.12.1-cp37-none-macosx_10_9_x86_64.whl", hash = "sha256:8a34a2fbbaa07c921e1b203f59d3d6e00ed379f2b384445773bd14e328a5b6c8"}, + {file = "torch-1.12.1-cp37-none-macosx_11_0_arm64.whl", hash = "sha256:42f639501928caabb9d1d55ddd17f07cd694de146686c24489ab8c615c2871f2"}, + {file = "torch-1.12.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:0b44601ec56f7dd44ad8afc00846051162ef9c26a8579dda0a02194327f2d55e"}, + {file = "torch-1.12.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:cd26d8c5640c3a28c526d41ccdca14cf1cbca0d0f2e14e8263a7ac17194ab1d2"}, + {file = "torch-1.12.1-cp38-cp38-win_amd64.whl", hash = "sha256:42e115dab26f60c29e298559dbec88444175528b729ae994ec4c65d56fe267dd"}, + {file = "torch-1.12.1-cp38-none-macosx_10_9_x86_64.whl", hash = "sha256:a8320ba9ad87e80ca5a6a016e46ada4d1ba0c54626e135d99b2129a4541c509d"}, + {file = "torch-1.12.1-cp38-none-macosx_11_0_arm64.whl", hash = "sha256:03e31c37711db2cd201e02de5826de875529e45a55631d317aadce2f1ed45aa8"}, + {file = "torch-1.12.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:9b356aea223772cd754edb4d9ecf2a025909b8615a7668ac7d5130f86e7ec421"}, + {file = "torch-1.12.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:6cf6f54b43c0c30335428195589bd00e764a6d27f3b9ba637aaa8c11aaf93073"}, + {file = "torch-1.12.1-cp39-cp39-win_amd64.whl", hash = "sha256:f00c721f489089dc6364a01fd84906348fe02243d0af737f944fddb36003400d"}, + {file = "torch-1.12.1-cp39-none-macosx_10_9_x86_64.whl", hash = "sha256:bfec2843daa654f04fda23ba823af03e7b6f7650a873cdb726752d0e3718dada"}, + {file = "torch-1.12.1-cp39-none-macosx_11_0_arm64.whl", hash = "sha256:69fe2cae7c39ccadd65a123793d30e0db881f1c1927945519c5c17323131437e"}, +] +typer = [ + {file = "typer-0.6.1-py3-none-any.whl", hash = "sha256:54b19e5df18654070a82f8c2aa1da456a4ac16a2a83e6dcd9f170e291c56338e"}, + {file = "typer-0.6.1.tar.gz", hash = "sha256:2d5720a5e63f73eaf31edaa15f6ab87f35f0690f8ca233017d7d23d743a91d73"}, +] +typing-extensions = [ + {file = "typing_extensions-4.3.0-py3-none-any.whl", hash = "sha256:25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02"}, + {file = "typing_extensions-4.3.0.tar.gz", hash = "sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6"}, +] diff --git a/server/pyproject.toml b/server/pyproject.toml new file mode 100644 index 000000000..9d14ce6c3 --- /dev/null +++ b/server/pyproject.toml @@ -0,0 +1,21 @@ +[tool.poetry] +name = "bloom-inference" +version = "0.1.0" +description = "BLOOM Inference Python gRPC Server" +authors = ["Olivier Dehaene "] + +[tool.poetry.dependencies] +python = "^3.9" +protobuf = "^4.21.7" +grpcio = "^1.49.1" +torch = "^1.12.1" +typer = "^0.6.1" +grpcio-reflection = "^1.49.1" +accelerate = "^0.12.0" + +[tool.poetry.group.dev.dependencies] +grpcio-tools = "^1.49.1" + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api"