From f16f2f5ae15a1b728559e1ac1a8b840f15a54efd Mon Sep 17 00:00:00 2001 From: Olivier Dehaene Date: Tue, 18 Oct 2022 15:19:03 +0200 Subject: [PATCH] v0.1.0 --- .dockerignore | 2 +- .gitignore | 3 +- router/Cargo.lock => Cargo.lock | 214 ++++++++--- Cargo.toml | 11 + Dockerfile | 36 +- Makefile | 19 + README.md | 55 +-- aml/deployment.yaml | 2 +- assets/architecture.jpg | Bin 0 -> 135442 bytes launcher/Cargo.toml | 13 + launcher/src/main.rs | 358 ++++++++++++++++++ proto/generate.proto | 28 -- router/.gitignore | 1 - router/Cargo.toml | 11 +- router/client/Cargo.toml | 2 - router/client/src/client.rs | 44 +-- router/client/src/lib.rs | 7 +- router/client/src/sharded_client.rs | 96 ++--- router/src/batcher.rs | 167 +++++--- router/src/db.rs | 287 +++++++------- router/src/lib.rs | 64 +++- router/src/main.rs | 52 ++- router/src/server.rs | 164 ++++---- router/src/validation.rs | 131 +++++-- run.sh | 30 -- ...rust-toolchain.toml => rust-toolchain.toml | 0 server/.gitignore | 155 ++++++++ server/Makefile | 21 +- server/bloom_inference/cli.py | 54 +-- server/bloom_inference/model.py | 36 +- server/bloom_inference/pb/.gitignore | 2 +- server/bloom_inference/prepare_weights.py | 97 ++--- server/bloom_inference/server.py | 67 +--- server/bloom_inference/utils.py | 1 - server/poetry.lock | 2 +- server/pyproject.toml | 1 - 36 files changed, 1556 insertions(+), 677 deletions(-) rename router/Cargo.lock => Cargo.lock (91%) create mode 100644 Cargo.toml create mode 100644 Makefile create mode 100644 assets/architecture.jpg create mode 100644 launcher/Cargo.toml create mode 100644 launcher/src/main.rs delete mode 100644 router/.gitignore delete mode 100644 run.sh rename router/rust-toolchain.toml => rust-toolchain.toml (100%) create mode 100644 server/.gitignore diff --git a/.dockerignore b/.dockerignore index 38e8f8243..5aa1aa3a4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,2 @@ aml -router/target \ No newline at end of file +target \ No newline at end of file diff --git a/.gitignore b/.gitignore index 723ef36f4..ec376bb82 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -.idea \ No newline at end of file +.idea +target \ No newline at end of file diff --git a/router/Cargo.lock b/Cargo.lock similarity index 91% rename from router/Cargo.lock rename to Cargo.lock index eda429087..551f7aebc 100644 --- a/router/Cargo.lock +++ b/Cargo.lock @@ -55,9 +55,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.57" +version = "0.1.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76464446b8bc32758d7e88ee1a804d9914cd9b1cb264c029899680b0be29826f" +checksum = "1e805d94e6b5001b651426cf4cd446b1ab5f319d27bab5c644f61de0a804360c" dependencies = [ "proc-macro2", "quote", @@ -166,9 +166,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.11.0" +version = "3.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1ad822118d20d2c234f427000d5acc36eabe1e29a348c89b63dd60b13f28e5d" +checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" [[package]] name = "byteorder" @@ -255,9 +255,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.0.15" +version = "4.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bf8832993da70a4c6d13c581f4463c2bdda27b9bf1c5498dc4365543abe6d6f" +checksum = "06badb543e734a2d6568e19a40af66ed5364360b9226184926f89d229b4b4267" dependencies = [ "atty", "bitflags", @@ -391,6 +391,16 @@ dependencies = [ "typenum", ] +[[package]] +name = "ctrlc" +version = "3.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d91974fbbe88ec1df0c24a4f00f99583667a7e2e6272b2b92d294d81e462173" +dependencies = [ + "nix", + "winapi", +] + [[package]] name = "darling" version = "0.10.2" @@ -529,7 +539,7 @@ dependencies = [ "cfg-if", "libc", "redox_syscall", - "windows-sys", + "windows-sys 0.36.1", ] [[package]] @@ -936,9 +946,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8af84674fe1f223a982c933a0ee1086ac4d4052aa0fb8060c12c6ad838e754" +checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc" [[package]] name = "js-sys" @@ -957,9 +967,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.134" +version = "0.2.135" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "329c933548736bc49fd575ee68c89e8be4d260064184389a5b77517cddd99ffb" +checksum = "68783febc7782c6c5cb401fbda4de5a9898be1762314da0bb2c10ced61f18b0c" [[package]] name = "lock_api" @@ -1047,7 +1057,7 @@ dependencies = [ "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys", + "windows-sys 0.36.1", ] [[package]] @@ -1074,6 +1084,18 @@ dependencies = [ "tempfile", ] +[[package]] +name = "nix" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e322c04a9e3440c327fca7b6c8a63e6890a32fa2ad689db972425f07e0d22abb" +dependencies = [ + "autocfg", + "bitflags", + "cfg-if", + "libc", +] + [[package]] name = "nom" version = "7.1.1" @@ -1084,6 +1106,16 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + [[package]] name = "num_cpus" version = "1.13.1" @@ -1185,6 +1217,12 @@ version = "6.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ff7415e9ae3fff1225851df9e0d9e4e5479f947619774677a63572e55e80eff" +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + [[package]] name = "parking_lot" version = "0.12.1" @@ -1197,15 +1235,15 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929" +checksum = "4dc9e0dc2adc1c69d09143aff38d3d30c5c3f0df0dad82e6d25547af174ebec0" dependencies = [ "cfg-if", "libc", "redox_syscall", "smallvec", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -1300,9 +1338,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.46" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94e2ef8dbfc347b10c094890f778ee2e36ca9bb4262e86dc99cd217e35f3470b" +checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" dependencies = [ "unicode-ident", ] @@ -1530,7 +1568,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88d6731146462ea25d9244b2ed5fd1d716d25c52e4d54aa4fb0f3c4e9854dbe2" dependencies = [ "lazy_static", - "windows-sys", + "windows-sys 0.36.1", ] [[package]] @@ -1584,9 +1622,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.85" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e55a28e3aaef9d5ce0506d0a14dbba8054ddc7e499ef522dd8b26859ec9d4a44" +checksum = "41feea4228a6f1cd09ec7a3593a682276702cd67b5273544757dae23c096f074" dependencies = [ "itoa", "ryu", @@ -1625,6 +1663,15 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "signal-hook-registry" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" +dependencies = [ + "libc", +] + [[package]] name = "slab" version = "0.4.7" @@ -1681,10 +1728,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] -name = "syn" -version = "1.0.101" +name = "subprocess" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e90cde112c4b9690b8cbe810cba9ddd8bc1d7472e2cae317b69e9438c1cba7d2" +checksum = "0c2e86926081dda636c546d8c5e641661049d7562a68f5488be4a1f7f66f6086" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "syn" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fcd952facd492f9be3ef0d0b7032a6e442ee9b361d4acc2b1d0c4aaa5f613a1" dependencies = [ "proc-macro2", "quote", @@ -1741,13 +1798,24 @@ dependencies = [ "winapi", ] +[[package]] +name = "text-generation-launcher" +version = "0.1.0" +dependencies = [ + "clap 4.0.17", + "ctrlc", + "subprocess", + "tracing", + "tracing-subscriber", +] + [[package]] name = "text-generation-router" version = "0.1.0" dependencies = [ "axum", "bloom-inference-client", - "clap 4.0.15", + "clap 4.0.17", "futures", "parking_lot", "serde", @@ -1872,6 +1940,7 @@ dependencies = [ "num_cpus", "parking_lot", "pin-project-lite", + "signal-hook-registry", "socket2", "tokio-macros", "winapi", @@ -1910,9 +1979,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6edf2d6bc038a43d31353570e27270603f4648d18f5ed10c0e179abe43255af" +checksum = "d660770404473ccd7bc9f8b28494a811bc18542b915c0855c51e8f419d5223ce" dependencies = [ "futures-core", "pin-project-lite", @@ -2031,9 +2100,9 @@ dependencies = [ [[package]] name = "tower-layer" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "343bc9466d3fe6b0f960ef45960509f84480bf4fd96f92901afe7ff3df9d3a62" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" [[package]] name = "tower-service" @@ -2043,9 +2112,9 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.36" +version = "0.1.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fce9567bd60a67d08a16488756721ba392f24f29006402881e43b19aac64307" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ "cfg-if", "log", @@ -2056,9 +2125,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11c75893af559bc8e10716548bdef5cb2b983f8e637db9d0e15126b61b484ee2" +checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" dependencies = [ "proc-macro2", "quote", @@ -2067,9 +2136,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.29" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aeea4303076558a00714b823f9ad67d58a3bbda1df83d8827d21193156e22f7" +checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" dependencies = [ "once_cell", "valuable", @@ -2108,11 +2177,11 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.15" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60db860322da191b40952ad9affe65ea23e7dd6a5c442c2c42865810c6ab8e6b" +checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70" dependencies = [ - "ansi_term", + "nu-ansi-term", "sharded-slab", "smallvec", "thread_local", @@ -2140,9 +2209,9 @@ checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" [[package]] name = "unicode-ident" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcc811dc4066ac62f84f11307873c4850cb653bfa9b1719cee2bd2204a4bc5dd" +checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" [[package]] name = "unicode-normalization" @@ -2361,43 +2430,100 @@ version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" dependencies = [ - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_msvc", + "windows_aarch64_msvc 0.36.1", + "windows_i686_gnu 0.36.1", + "windows_i686_msvc 0.36.1", + "windows_x86_64_gnu 0.36.1", + "windows_x86_64_msvc 0.36.1", ] +[[package]] +name = "windows-sys" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc 0.42.0", + "windows_i686_gnu 0.42.0", + "windows_i686_msvc 0.42.0", + "windows_x86_64_gnu 0.42.0", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc 0.42.0", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" + [[package]] name = "windows_aarch64_msvc" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" + [[package]] name = "windows_i686_gnu" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" +[[package]] +name = "windows_i686_gnu" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" + [[package]] name = "windows_i686_msvc" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" +[[package]] +name = "windows_i686_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" + [[package]] name = "windows_x86_64_gnu" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" + [[package]] name = "windows_x86_64_msvc" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" + [[package]] name = "winreg" version = "0.10.1" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 000000000..d3f7dfb09 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,11 @@ +[workspace] +members = [ + "router", + "router/client", + "launcher" +] + +[profile.release] +debug = 1 +incremental = true +lto = "off" diff --git a/Dockerfile b/Dockerfile index 9b8a20549..70b91bc21 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.64 as builder +FROM rust:1.64 as router-builder WORKDIR /usr/src @@ -9,7 +9,17 @@ WORKDIR /usr/src/router RUN cargo install --path . -FROM nvidia/cuda:11.6.1-devel-ubuntu18.04 +FROM rust:1.64 as launcher-builder + +WORKDIR /usr/src + +COPY launcher launcher + +WORKDIR /usr/src/launcher + +RUN cargo install --path . + +FROM nvidia/cuda:11.8.0-devel-ubuntu22.04 ENV LANG=C.UTF-8 \ LC_ALL=C.UTF-8 \ @@ -34,17 +44,15 @@ RUN cd ~ && \ bash ./Miniconda3-latest-Linux-x86_64.sh -bf -p /opt/miniconda && \ conda create -n text-generation python=3.9 -y +WORKDIR /usr/src + +COPY server/Makefile server/Makefile + # Install specific version of torch -RUN /opt/miniconda/envs/text-generation/bin/pip install torch --extra-index-url https://download.pytorch.org/whl/cu116 --no-cache-dir +RUN cd server && make install-torch # Install specific version of transformers -RUN wget https://github.com/huggingface/transformers/archive/46d37bece7d3ffdef97b1ee4a3170c0a0627d921.zip && \ - unzip 46d37bece7d3ffdef97b1ee4a3170c0a0627d921.zip && \ - rm 46d37bece7d3ffdef97b1ee4a3170c0a0627d921.zip && \ - cd transformers-46d37bece7d3ffdef97b1ee4a3170c0a0627d921 && \ - /opt/miniconda/envs/text-generation/bin/python setup.py install - -WORKDIR /usr/src +RUN cd server && make install-transformers # Install server COPY server server @@ -52,9 +60,7 @@ RUN cd server && \ /opt/miniconda/envs/text-generation/bin/pip install . --no-cache-dir # Install router -COPY --from=builder /usr/local/cargo/bin/text-generation-router /usr/local/bin/text-generation-router +COPY --from=router-builder /usr/local/cargo/bin/text-generation-router /usr/local/bin/text-generation-router +COPY --from=launcher-builder /usr/local/cargo/bin/text-generation-launcher /usr/local/bin/text-generation-launcher -COPY run.sh . -RUN chmod +x run.sh - -CMD ["./run.sh"] \ No newline at end of file +CMD text-generation-launcher --model-name $MODEL_NAME --num-shard $NUM_GPUS --shard-directory $MODEL_BASE_PATH \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..3a80a12db --- /dev/null +++ b/Makefile @@ -0,0 +1,19 @@ +install-server: + cd server && make pip-install + +install-router: + cd router && cargo install --path . + +install-launcher: + cd launcher && cargo install --path . + +install: + make install-server + make install-router + make install-launcher + +run-bloom-560m: + text-generation-launcher --model-name bigscience/bloom-560m --shard-directory /tmp/models --num-shard 2 + +run-bloom: + text-generation-launcher --model-name bigscience/bloom --shard-directory /tmp/models --num-shard 8 diff --git a/README.md b/README.md index 6d23d9c5b..18e9865de 100644 --- a/README.md +++ b/README.md @@ -1,50 +1,51 @@ -# Text Generation Inference +# LLM Text Generation Inference -A Rust and gRPC server for text generation inference. +
-## Load Tests +![architecture](assets/architecture.jpg) + +
+ +A Rust and gRPC server for large language models text generation inference. + +## Load Tests for BLOOM See `k6/load_test.js` -We send the default examples with a 1 second delay between each request. +We send the default examples with a 1 second delay between requests. Stages: -- Ramp up to 50 concurrent requests per second in 1min -- Ramp up from 50 to 100 concurrent requests per second in 2min -- Ramp down to 0 concurrent requests per second in 1min +- Ramp up to 50 vus in 1min +- Ramp up from 50 to 100 vus in 2min +- Ramp down to 0 vus in 1min -| | avg | min | med | max | p(90) | p(95) | RPS | -|------------------------|-----------|-----------|-----------|------------|-----------|-----------|----------| -| Original code | 8.9s | 1s | 9.12s | 16.69s | 13.7s | 14.26s | 5.9 | -| ISO with original code | 8.88s | 959.53ms | 8.89s | 17.08s | 13.34s | 14.12s | 5.94 | -| New batching logic | **5.44s** | **1.27s** | **5.28s** | **13.12s** | **7.78s** | **8.92s** | **9.08** | +| | avg | min | med | max | p(90) | p(95) | RPS | +|--------------------------------------------------------------|-----------|--------------|-----------|------------|-----------|-----------|----------| +| [Original code](https://github.com/huggingface/transformers_bloom_parallel) | 8.9s | 1s | 9.12s | 16.69s | 13.7s | 14.26s | 5.9 | +| ISO with original code | 8.88s | **959.53ms** | 8.89s | 17.08s | 13.34s | 14.12s | 5.94 | +| New batching logic | **5.44s** | 1.27s | **5.28s** | **13.12s** | **7.78s** | **8.92s** | **9.08** | ## Install ```shell -cd server -pip install . +make install ``` -``` -cd router -cargo build --release -``` - -## Run +## Run ```shell -python server/bloom_inference/main.py bigscience/bloom --num-gpus 8 --shard-directory /dev/shm/models +make run-bloom-560m ``` +## Test + ```shell -./router/target/release/router +curl 127.0.0.1:3000/generate \ + -X POST \ + -d '{"inputs":"Testing API","parameters":{"max_new_tokens":9}}' \ + -H 'Content-Type: application/json' ``` ## TODO: -- [ ] Add docstrings + comments everywhere as the codebase is fairly complicated -- [ ] Add tests -- [ ] Add shutdown logic in router and server -- [ ] Improve multi-processing logic in server -- [ ] Improve past key layer indexing? \ No newline at end of file +- [ ] Add tests for the `server/model` logic \ No newline at end of file diff --git a/aml/deployment.yaml b/aml/deployment.yaml index be28ceef2..35d19006f 100644 --- a/aml/deployment.yaml +++ b/aml/deployment.yaml @@ -8,7 +8,7 @@ environment_variables: MODEL_NAME: bigscience/bloom NUM_GPUS: 8 environment: - image: db4c2190dd824d1f950f5d1555fbadf0.azurecr.io/text-generation:0.3 + image: db4c2190dd824d1f950f5d1555fbadf0.azurecr.io/text-generation-inference:0.2 inference_config: liveness_route: port: 3000 diff --git a/assets/architecture.jpg b/assets/architecture.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e0a5f7c6ffbb34de6ee877bdf2d74fd3581c75da GIT binary patch literal 135442 zcmeFZ2Ut_xmN$No5~>lTw;)B7suTe!!A21w6lqeT(m_C_DuEzMZvp}WQWO#C(mRok z^dh~6BGMBO2#|z-oS8fG-aGHTpYJ;}|2xm~eTRpH=dkx#Wv{*WTEDgSIUYNn15Vvk z(^Lb<$N+$h^aC7E0@ncY6F;w?pX8*Af{NnjMNLITNkv0VOG`sdLqkgsrl+MlNk>D& zz{qeCeCibADO!4_(@dvMlm0&S^C4tEdy-R7lRBQFqoE_+{e#!>XMmCVgb`H`IT<5x zf{~1zk?gnyfROm4B>R^O__vGf1UUsI6^NRKmX6e*{1kA4MEe8Hh!) zBPG+Bi&v;l>)iwKKVp`87W0-`;A(Lri~aycQ2PGk02*3WHg*n9p|is0&Wp&%UXqhn zxP0yU4K;NQ&6@_d4ULRVOz&7(+t}LKJ2<+zdw6Mj8QWeGJR}#{;yif--0!>`&4DMY3NLEZ|Q`_V0rIhg=f?JvkX^@W>ef zC_pgCIN?tVoCE&f*Z)(&|GieA;L3u(jk`B@)wpezvF;*zPcOFSLy@Z^@;tXLDA^HS zDY+h(rGd3Tcw#1BtvbJtMI*_XEKX^{{AEEURp7V!z z{&UYc`=|8t=V<-8vH0iG`fu0IKeUaoW8nXHWBnB)`-e>bnF;pK9n}90L!_*Q;m)P6 zJNQHkFIc==Ci*yVM!*kwsxnT7s}agkHVmFN0qqNYFEVYvyGp8bpu%sR0{8kWUu#xT zz!*o~l~ZBB289+~rCKP%32d7oCj|C3aeJQ;EI4o$LbhKREj!E6 zv7unFZ>^DZdoId5$nY3&^yfz2Pme+}z3w}@N*wc(8Jr0DM7A{aA1)A0AxD^t zsyuQ*(76;;2|V~_m)D%)^ivOoopKAyr7tR30`3t+QB~foEWd7KX+k*dm23nJhKb_) z7JALp8gJ2GjMoNSC-7%*u=|(B5f?D=RX@USRKONrky#O05}T2S!ZV!5K;tb4{sw8o z$#38~1_o3xqeR-f$H4qK;1~$B!ORjv_k&Ov1JE(Rs0zZ1LW7QhAFw*vV<6x)6c+|S z(WVHwW8j@Hx(LtQcnq}Yl6ETEE;Nd8dW!)$_W?}g_Xp#QkPM_1Bi-BS7@)j>#4-cO zrc#{iF>w8;8IC*u{1^zyApbp!@4mk^@84+Y-^}}W2JSy=>Cc+?@38dm zGwv^1`bR8@{QVjCZ(5oq(q@nr0sEhefZd;51pZn}=cn-;IOs>a4bt*};BiZ9@C)VuBJ6Os`(T%30$pQ(VzN)yWm?v}?9iC1%H3xFraA*i`zNJ(#j+{6zCa0@SU z4anW?`^3~_rt6mEgi=4a|Gxjl1wuT^DLSxc8Bb3r_Ha6~uL{<{rdHSapCf{0>*h$J z6f-T915&U_;k`^8D-pJF;9cS^BU}H!WfLTugO@=!(lLp9h$Mup zhj?EY!quZ(tWNx0^yO_$2*Ko4M94L!s$TQw@oE`h>U1=0BlXB4uokzxV1)_)%x3iv zli&~A3+xugxxkf}sygCdQ$bdle`=cQ7_dP;L!J=72M4@QwkfM!3>uNtHD6`ozMHPx zW}rNLspv?CY)$5$>TY-ow?&Ud&w<(^599X(PdEGVGkj|;#395}2fN+`Kg{fUwRqA( z>ru1h9bh}f7rr+|h}LZ=XbWX!=Kpp#i_%$V=SWez_Nx*~ zdi|5;rS1jKVPTGq!QvLfJ!U(j(Iss{=VGsU@#A&xM7%8RG0-1{#Mm4I6L~w)xRBXh z(oP(A4D1ER5ef)>?L+|CtXo8&UOon>;Dp@Yws`i38qkoOmOuRj9MqvT%^Q6TybvsU zjdmkszOh~)J~4<6sq2x= zYdll$(V1IVOy4TLMsZWBce903S3&TNE~uG!fsoT6O&p#%-+QP`N|9O%#&yi^SVOZh9u&4Jb^zAX4CaAbg;|2M#SG7q)7- z=E3w3eLz3wqHyry`63@XX;_Z|XGta;yzM2PictK6T8U-BAmqpFDT&h>qipiVOuijS z^;gEsM6PRxbb!Wg|5MkF|HB>u0?nf$Eh#ajvc)Lpn;SYK0WCC_-azUW1Gys@W7VD- zcYQO5C)gA;r=(eA7R~JK4wsKD6hvogX-yw+U!P^iJI`0$x&!mK30AMBqRaabW!myc zu5+E|YVI7<<16Icqdi^W(qg@r-UgPRcQ}7XzvQW$d#sVgmR00}XSFDr4`q(FrSt1;{mmx~kK^BgSPvAYkB^O^UGY&eR+D+UX{y5+3B zO%Qj7(*~(w;jO;<%UT1nsrTIVkSv)~S!El}k1osPuuDDQACEg1b?ve>c9w@--GY5JD4nrM!p?4lIR=kE@x3+i@$ zZ+We+ORGXoiqvZGf}`cIKtI+|cbd6+tah0ZQF{1&%mXom;>U~IL9Ov`v|I+CpK%ij z$?zUeT4cKJ53$7a;+8SX3$4K+<>^73&5nn#pxtqVvc$X4aG%=O837Kw)1^uL5c}Iq zcd4xJ*Tz|W0wMp2`xIF-{w_`uv$jz5d{c=9E1(*#SNw`K;O>bj(@r&te$9xsg|83Q z={Ynh$iE?jp{#yF#1<9Fp$1yg23&V|3^dj2=sBPvyqDt$Z;%0lKei23S7JTM;>Foj zbV1XndRSUCm6+bgXuKU5cxI$8MJ@0#1JwV&693@f6~-Ec&Jjektt{$H&gd|zON#DY zw_HaaE(52{ITGxGI_;IzM2z6!zOk#v0D3y%gJIs)7f(L)ofxk3Sm=x79K)c3_!3Hd$&NVW9m=kbK|L>-=Sz`6<}fPfoGmcfOaayDAeVMdh4uLoG(_u<@Y* zq*>~>)H!(IF+lC!u?uH0pR|~Ld9%;`+4z`M$i3ya?ecB^rI|RlgA8hAxtF13v6Y*qyLhrpLb4#s+3!i7jqB{Ho zKMO8aP$?8%uCr6>>0$_Iu4K!M8yR?dX;53#>v8d$;>~ij4yPBy?dTHj=3%RCf1Tb( zQ4MmzLH0s^n>|`KsKu~{JURKD*aX(P^6^?tKW^0P8DpKBm^gf|nk_C9!@sB0E9q_u zaSSls5EOoMcdmtSWF0*ZjP`w5HfJ-m*wJ_GX*}=>UmM(8jZ=V+NF8`Zlbi)$*X%r= z20PTuwds5%nX%&f6lKq=7f8CN+}4nqu|?}sD)SNB4bqK1*!_gHjVld(rQi2Fhb?64 z*=rO6m5mQFmr)@QaCvm_MCF(Vc=k?TLRqb0)2+-&oQbuKk6jht=#uG-dPv9Gk;p_V zUdl}NP-QJUxztZWfI&&AK)m)%xp%gLax@-kb_|T(Cf+ZEFmZdSPwf&e_)xSmm3S{lk%Ss8F4 zrj^s5g+-k<0|2;zb0=DVI#Qr^NCj+T$L2_38s|bX(uQKpLigwmz7Gq82?D>s$_&g`sPmWTu=NK%}D0TSDdZgoovidg2vz#DoezLfd z$o*0w$fShc2&?NZ^ZX!R0;z^DURJ4i?Ag&L%R&{P~R9SLvHWAog~;MF2_$3zEo@g+b`5+NYf=Dq)>lYNXvLT0rLg zgrTuEv}o-Us`OH6HkYzl^Ec{D6qESaD{z|qCpJ)2$jmXY-9P7%+1zc&dvnZ8NHGVV zXJ$4ic?{g#rO$m-_I4DxHnb}b4!vM`2X9yv-c~7RY;+Y1@!WbnR=P;>or*c8o$J|F z@_`Fq;7!x1g!@yO-Sxhyo3RHD9}=ru!*t|>H$05-;Pw&?BbUiRG#c3=I!m^O z>E5|{XxL0qUrFy758duRnUYvk)v*PSc)0n*K&bWnrD?hMiHSa5@Th4RIbEptli6&8 zYFs-udDJ*)xp9Wo#O1*vVS0Lj&*Rne)MD~BG6?dk%~L9F#4FdJNGQE@295{QyVdkba{|HP>w_>y*;O5Dhk~h_5-~cGK(%g2BO!HhpXw`2Ei=> zikX{FiLX`a{$SQ!g@y1QZ!-^@SYD7PM60kk#@)fmsU|3*LND(YAE>H#6svtVky_yX zrq@A7w@AJgO^t&!&ro1`ljgQ)RvxZQs6;>fPb3DDN zDm1Mx`p)ih8<-oHw-<`pZ*n$h@?*lP)pZFxDV;UFk=I3?{0&l3XCEmOWmC2NDa+`V z$cCOOhb24i7{D$xa`IFLDRE#++iR#IZBXkIo{95XU1s)PwTkOu3MV9%ZLOYtxepWU zr^?358gE=V2Gq}BwT=Ng26d7?e104u`(iCgfI|dfMlIYaTgzNAG>W zl36KO|?LI?#Md?B=KQtP0#H^o(=;6ac^fl$4Pr|*Y}yAhwyj{EhRoC&+4 z3aL(~MHtK%aU&Ek`w>b`a|sx~(gKsL-QE}but#Up1P@}*Tg(T(cuVVgjq}yxFFVdt zs+VhkPOS9|UnQoZ^buOV;Qh$_&?e-T)_c3a!j=d3m_$lq1NRoGC?e*f%E)%M*3Td5 zdS~vXV%S1FT)9^cm80r2R_6ybWU8#gBukZU4k(xkdeWQU0?Qw$^VpM%?qLXpN@hnF zR~nc~a6VbzC!*@-6~hy~%jJcVzV3xi}+i3V%Oo!V|mG>T2eJ=|ao#{93M@>i~@;X@Z zRu7e%h!=WHrc6G`st%+{FdgBK&3(&SOhp;rV8=m0e+r!Y5y$|)Mm>N}IJM7~?uYv^ z+gs8N$%xh>xg9H6?}R%VZFgYXxhulx)f9Bkw>B?+atUt-@j-O{#VbcxVB7bmaq*9^ zPq+4b2(KErn&*!J))jqC)*N5&tc`-{3xc=x_(ej-1C&Z?J|YuA0jl!pOn!su7`^mo zcaZqA#{h>4HO|cdQ&4-p!AC7LnL_ZSb$J?0IF@QZGEvYY16c41VykK{4n1Qx=itfOF*a|Kk`7xj!jbGi!oC{J0#`8SplsL*>;c z36+k|vWjbT5w#&Cs7)Wn!S$*njlCcAn%EBELRg)BoA1{(%d>nQEBTq4m9CmSrfB8b z3DK&l?RDmi`+oOnix290G9b7ApxgNO(GKqAFroln3Bzsa6@_?$o;}(2&}((#J?rzd zH=SLo+(md9()=4DOS>~CbnY&dlDi#SAO34XpbZv^xuq{h=FoN6GUDs%7c6T{rdN73V*-c7Ang>Hzw_u~~mxB;XmXjJl2E78xr1Rx5D zcHZl0VBv0uTA2l|%_8@@EUsOc_JfHM1&8}0{cz*ly2RaK_Jm`glvKKa z0`=9!x9e8nvF&{5Nr2mTP#Ufk_+>kar9xQhW45VdqRU9smPMAR=%iiFC@u2^OM=}K zqy^L<9NXRsXF=%H@^WInr9WbL(R_sPXM#+~s#rUmb??5~(& zM}`%a{4AoV@7^7W$`ZRv0&E7%gg30IwW%ab68wy{MnJvLyH~z&tBsROX)|nUw~IGw z7WaL5>!JEr%ob%c^@#Arn+7X9gS+2n=jwhqmC_}C^x5_BjT)Vo48~LDelMBvMvG?z zP@Hi!tEvpNv47g;_oZRH}RRdu9=syx4-wY73`;< zn`+N$oZaTpF!eFObb&_TEcy$OPDGiXe2(cH$mG6e;yERm>DW8tboj#zA}3)8 zb+g*2A;zeb{UK);Ohza}+M7lD!)cQ0#6GOGTZ*S2ftxDygp)*PGy|nY9-CZeKXu`X zXDDnogZdb-p7p;pTRPjM1!0&enf6XMblre4%_i*dej{EQ{<$N+G9z0vGvWVyZCjXSYeDQ`M8W1w0>$e z7g5oY&&r{yB0}nj!1G{{8_(yUj0h&|a$7p11BzA&lxyO|gtKg-)|%<_OiVFStfVZM z*SxpG$y3Bt7S*VK?VBz+YOWN%?as?aN?_A=9RoU-0|9U0_2MI2ye=UIi54cM=Up>! zW+Z?1MbSFknK)HmmC|)2nZ9)3cMEF;wf7yLej{_z6mz5fdIKY(yk3GoA?FgHrw|PD zuH!QhYI`U%-hkw`G2)ONDe%(K%-6_!PdfW2O(xjHA>*uy{8lE5F8S-9l9XNnM^Yqo z9Zw|0iIdPGEx%bP2a!&SkPgE#>pSkzM39HFexrcvezLqJO>ErHpyR9$%-&!jf&9`b z*T0@&Yv96}qQ}ovhX&nGxW-7YOir_hK<`>#*Jg=ZFbh(X#@w##x**G($_IH>G0C= zPkio7(R!*(t0->LXt+)+EVzdHaS*#3#7i--XLTHBD!(GsM&>nL(%tx+TTO%ZH0kYA zq&{}{bm)>9Ti{BM4ESYyWFVc=Bpt@T;gr!$B~{+!w3h_Kas z<{%1xQi1^|dbaXW=4v`QUj4nG@#%5P)zOOF&m|h)Cwwx_Xb3WauAWrvrh~9%RNz|SPmGtSGd2J)0nCJ^>8X%Nk8@w!s@PWuA8RlW8 z=p)Iv!lwOU?b6;{NJQf`!Qvpn7w&f-YBp1G$W!?H3H~cZlGsz=SJ64u;EwhG4QCuawJ`I z;e=MUrP-R%^~o>X9eSg^{vB^`7IUTF-E9SYj2n-E;8}`ez?L)zo~CX!M_%j1=$b{J zwt!#C7%i5$Pq`y(Ds=@W!UC1#wyOsaZ|5M~#BpRLZ(gByI~vtu;&&RaFtnN64%^;% zJ~gFLfp0LtE1VDAwjM|KU$>@aiTTmv^&=0fhp z*lSBoFLDd#1|<_ZS1l~QZl1fs#}2K4Z?GMy?h5m21(r30;CPafgAZYi-rVJcCHvnnO*t=23@_;Dd_1f!wNsVXb^?#F<{m~u}`c+isn%qB~s56GnP%xxx8 zT%m=jU6|dV!_e&o4?^uA6LFTMShbXe6XEE-)7NM-IjmL8&)v-3WZ*w|P?$A5tuz0E ziL8e>0xgrh1FCVeNWzV;7fU#D4 zLAdxFVjr}`=wr>L)VErT-p%tg7GJC%8)lU5BJw`kC?>sHKbLExV3?aDT4Ez%(M2Qy zAOYUQzRTx)VeL;n3$UOTIrZ-PVf>YxTidYnM?J+n#aUVf^#DhpvM4=Y-$da38xj7UK+rMp zyA%ZeS{J{e;Qjj)en(0FD)Zkw>F;XwFD#M#)8BH`@*lGOB4xI}Xz6b$|2<{=dxyU@ zfM4i<`mM$MR!gqGI}U%VrN0Or{P)J;4EhgJ_`4(h*J}NHru66+Wd5sJ@>`jI*)o2| zikJUmzWw5C_>E=!mDMjz6(NsK5<58~7@QwrjBnhyd*y10B@(mHZoqUlo?7 zw(G6->~M`oVqb(?B*%8zCfYKy#;dEpHt?)Tr{rmUFHNE>7GyjNA}8EiW;*@CCE295 z7PciJds+HB>N%XvkHc>aN@Lt|$Ky>5*_raMKe~b-j)Si8PqfEY$!*N(rP5RnUW&Dg zH``Wu1-G56!eRoi zSH!;gkG{@F@Yna{z2wn#>LVp1%e>g($+FDRCEub2C&_aFch6I;LCup=$1Aj?e zbDA)w`I@hL!*%_};7AvCG<)XUhrufyVrue*hf2=$zH%?x2FjM+IT?2U(CIT7U-PMO z;%+pw?0xS+&>-<;i12vBZi@@J{(!b(!}5Co*z(ejKF%)meYK?|?C=1(&Gr==tqyIC zv!5-!eQ~1>>lH>vmXIuZeL|(diHH)$jl70|UpX5<;KMhN8zf*XhFhxENgwWYa14CD z+y!SwaNzFQ?$zkL&CZNhY)z7r?o_w+=u_yUSLGev)bIB83U;Wvw@g|$y=LMn)rN`R z1})WZIifG*4md64wBgeC64F#L-q%QO@0|JUn~ICQ0tWtW{Rsk5Z?HQD(IL|DX#XtN zEiELT(gBCMmQZNF_r8IYcv5-}$Eu*_Q<{-QtEDqty|Wh^ZtodFM56CfB2LO(XtkA9 zE);Qr^<}>sN5m?1#@iY0XisHX9RrY&V_?p zAL>9yMhNSHhBJ7jK@WnW&QP7S>$@)2a#wokq=lNA*Cin~@t)3Asx)^Wb49)Ize5P| zQ}QDPdK4%$3HS_(!ATzjH2ZsfdtQ5Tc!L_xygfoXJHhHUrm+4)MQYy4q5GW5l1D68eBiWIf=P$F#rlP~ zt{|=uKJKlJz@wY1>1D@2TdHfjuKC*D3=10>Lh+sB)8{(L?kA}4d1*Ym_<1MABjI{9 zpW@xJ6jE$AwuKXJ4^k1q$?e_2R0gYvn&9q?IKgGhWc!6G5)@>D`89-+&Gbixv;~w( zo)k;d2zaNCe{eAI_l1yrI0X2}+F}AhODOZBBMunf8adR-y}r!)c3xX_-fM$yo}G5pPX#L$mOq6Lx#K_!*-UaV_4{<6{Oi@~2^oQT_B@lVj&r}tv9cFjCKsx{#ohn6pQ zGA)5YwIP+RkV@u@SQbSj5`bWQ6q zG^x4qH%VBFmYWA0sd8b{Qz*#cL|Orrcr47AihWS5_O$bv3|kMHNDeY!r5u zkSKEv=ar}oGjWeJYhfK2zivz&$=SV0wmYfBN1z0JF_RUkRlV|kxB5X{@DkWYyyOPU zJdz!uh}{o#pQZIqY~H2}zsaViULC@w6*w`+YM%<+1!;on0S_dJRcrY8^LbEEd89#G z8b!a7qM+yCl=eJ>%DEp08yfnzw)mZmC#aB5IlLEjL2bz5`29${<`E;Yzk$A@y0&IW zrswiTnIA{v6R{aUX;^tX`5TiHp_Orx#o!p&(?X66292Zn9C-D#!ee zr8iO|7JaWQ@k~4>QSbRTQBk|QUpf|ZGENq?AxohFco237JrYL>*U?y`>IK2C`9*cg z*^bGw-!erjC*rzp#Pz*W6->C0Q9&k}_DGcn*5+g{ftt%j(ekHVSgoPZ2at6!tMu6z zK1h?l%6F`KuuM9;x04|zq5gq{%(P>tugbOn{$Xbz50b@76%v=mfkPJ%yHR8+r0Lzj zhq{Q1YBs)X8KlMHaKBcZVKF8uH-P;`uKJV41tou9XEf~DVR};^P8bqWM`ve`!96#? z*kF)Y}*r^Tj}w2OdF+m!m0h;Dz+&0^tSWjmG%`w>HfjQ@FQ} z&ak^vkD9S-xo|=l5%G>pusD$N6OI0pR=}MoA0Qfy6Gi$jB$0pYPyGy?v5-1=MWi6d z>4Ij<{wL~6$hM6?Ya-D}C?`_gX|MFT3XE|}8Ilbr{`7{kH#4EI2Xv)Lfo}X>cx^vu z;_s)Pe*Lr}FT>G8yv0`{tzS1$66^QOb7W$;`{K%F>&}M*u*dV#{At}WNrG^9W-{YF zbKJ!b2pZgZ^en}l@sRSf6J!hj4HFrvMd;VT-7d3h9ynyhIPOE zL;Ku64h0Xcc6R&q!7DezCa8~u0~?fA67^Ou`BMhUrpQV?=>%afEkd>|G<+hSn%sQT zU;5oud)nm4v8IO}#_~PcGds)D9QNMKV&B?$+S$(H!-^X&A7pdd9y#YV1ozmwMs-s|F)2IW6R#Z_FL=$v50S?}JQzeM zI+_iQ`REOw_biZ1&-M0c*^jm2+7x#YeDKufS>fda&qra2%1>pjkE(i5tsI>R{4U$! z$g_qd=myjd{UNlLj+0D0214_Z#cq;^pqyjCUke|4Sns-obOYm55gy0DRohxpl)8e# zB}0GH=K1O_Du&PvMW4Wq5@?dQh;$!*9$zB1cB8)YW)sOIx50TMMD=|sCrL6K+Lg~1 z<5`s}q^~)&^qI6%d**UnvVM;mUq#L_9Rp|bm7VaCn|9-d1bSbMKalR8t{H*X{w6j^s3vpU=2b=fHJNx`^z zifIKKEnjV{tZ61v#FYOa37=7$d^LZOe3Zz8U?hYgyui)lE_m`IL;DfbDcKar$vbPt zj4Q%dIpdpO-_o7s@?fKl6ARP0@wm8SlKlKl;E0XLO+ubYXFCpl)?82}dmRIO{h)zk z;1tRROsWSxjkF`dXAW4UfoWa*CSrb_I<@VW%D1g-BP#2!Xa~T_k zVpsWYn~Qt|xPFi-6+`_-kfmqSj|>pX#Tf0rFesX*X2?ThSJC;piEnvYZrFhXxc zg&jAzGU2pVpmHgJD@TLujmQ_S>wS3d`vqS_V_W?1;iQQ)D<4-;jPA3c>1Ek%1grVL zZ&3sF38|0B)gAMLMM~IWy3}hKmNFt(A(!Sf_;{m_jENn0yfzGSyjh*$ZAI@)ywz4$ z&GJl0FpzRNwVl&r(inW*)|q@PcaKPXv_0qVJ-m-U9XB8O?Uf3b+{TmnY@8Dp8E>%c z`=~D`DvZFp^HLdP&&$s7G4cSuUYHa#sq~1}8+;4rHYcvU;T|b&X7~DNDOv4)pI9_r zr(09lU;gKdczApPMB>p6WTO&-5}V$}gujKc3zx)G&L1UF>>GRPKB?H31|K{a2vXD3 zQE-10_k|)>1C;cI2&4_E{a5yJLPzsiGyJ_p5b0bqJG26Ki`jJGb=W#7mt%h*VT_zm8Ym^IxJ8RMEL(a5`!&D;Y{7V)5)Fx%Et81D4(_*R z!YZGw3ysJ?*WXI57Co44l}m0g@lKHM7@Iol)(1cg~yS3hh9le9IJgm60rWF867>gaBh=_g5ctVr0=1YJ#-(BEj`kdUlzz4h>?$t5C zNfgCv4f_dsFG?#H?Bdnr3*HpYN;}m>3x-u23I%*r;ya@Ota(I_-*%iW?+rY<4dFR} zkJb2$MbaHfdZpBi`1q1CDLZPjyGO-K>9r9%FUhw4XWWn-xs8pv0;4swc5quXRZev4 zEOog&Lhr+^%(xSm3AvX@?{l$XvA#ETE$T%-Xb^f5;gN~Z#;^qw&v-xCcbMTH%J}Ns zc=x7Q2$z)=?p5r2PmvpdyZe9HV*<6$Vs3k9rtEnPpySB}a6>nZ4Eh5~Pn^Ffz2b=J z)g`$bN(=MIHhDZ*?#c|u0A3t9uZ`4;m^z#8P_5zbabkH4V^dc-+X@zc}zRb-j3g`jDKw#4w4_p1}W4(sP*luT&{%EVZj@Zu<&ccg=K4f zRQ5I2#VUSZeC-|!hqAk<9>hHMj8~YDe&bJ8l>fHzz}eC9VcvP+Vfpd>is+&gCC<8Q z5!rGa23lXTk!g9`qTk__pfGWdQwTFO&)LmZe|?z+p}Gz)`U>kOESTB`c%m@m!p9bj z5l=HA?folMy)BBOV6EuwZSRDV#I3u@+co%2U)WqUhkg1fglM*RdvnRH3BN23Z~XU8 zjfi}`f#Ci4;a77nH97?ErU$g4Mdn1iz-IH&uJzF5(H)f{hL9kU*aDcXi}A%zf7x;W z?9mQT=WKc7MZbP%385P)mbVbfPk(vcvw|LaO0*JGd%!ug>YHQv?9Y<{HP4pay0*}_ zF^#F5OSparZuS9T)2i7OjNa6|&h+@MrvKFqF9*i?`$qmI^Fg)%EpuJZ#%%3nl3XqQ zzIo5``S^i4QNP3JJ&B|oIwx@*wXutywR9Ckr8aixBl?M$TEL@5 z9nVs`hr-Yzn|KzY>2Xk_;U7*p1s>2u)iNV^cr(U{}RLJig{HA zDv+SAk-iop)b-IWsyOnLu*84tZT%`R?DIBURHeA>YXnJ9>g$Ymc;qbG8kOmtc*^8GgF=|7+fsfCQ0>uO zBycJxL0Hh==MSGh`}Gjy*IJSp7!8&Yyl&PH$U5DwF8(UMSY932-X*4g-q>LMb_}i` z)CQJj zYNBf|Yfy#B+jIGPf6@LL8`nnHm2;+s*zQNX-BNO-y;H26eE$X6>fKMF69tCZH)e^T z-3n4YWxDMymMOt3;pPx&Z@Mv#^ciw4^Lo@zNI*GMH(a#G*!w9C75sp)h?&iW^rdf{ zd&Oi;SA6v*vx|MZUV<)Q6S=Uxf*!4S;SAcTEJ}2(K3Hg0y7!_xukJ&+hthSnmPC;p z=L5ru8aUG(WpiT@N9a}KSH54CRU~i+nB~Sa=pd1Eb)uL!`8Cra!ZFyI(I4Esyd)Wr zZ$(jctZ$8him zLBjX(F`!>wF;q}6vFY^X!|i}`EM78^XA})j1ZbW1r*cZ}-COk$dGu%*roA=T&O-9r zXfR-6c(6YdlSSH<7vi%&<>8Aqr4EO|%kZa{>cp{%=%OgevqN|mEM2n-Url9nZvum) zQDtqc{5I8|$<)hgK5J~Oh>gy}5zTk#JE`jSnou?OVX9zK!mar#f^^RB9eNIoKmP!Q zTXI?_Rh@_)MW~ctI~bjbgq=i~;S3fSwB6yu|?F_*>) zH8pA_TpBKd%1&Ws!=bbdD&8r#NoC#MrJpM*?wa`)?iRQf)O~yx0A$7A>RzVd)%YT~ z%p3pXz|#5EJ%^#s=+(-C_^*YNz6bi#X=fG5UK_u`8s=mTkLTIf!>=O<9&V$_hemxo z0voeq`cH&$H}YIP??FN~S*|EdsPsH|sH`AJJd3)xAd{fNO1yEZX@k^yO`+^LqlE7`>Ftw{mnx@c_G4k z(#a^JyLs3h^cTT7viTHV(tG9LB{?LUgNUwsTJmCz1MwEN9uQjZIIWQamWrewYaN?b zpdokbh_7V}RTW(G(QMwZMot5#1{thMT!usL+G$J2#Ns=jc^X!e1n*WW2R(UvO!`KvFtL>4ze0Bm)^l*WDVJ32%uMxMhRr8<@n_EgHwxa&Kqz+@VVT{**^Gq;PZNfw)b5Lv`51426`IOd+Fyu9WF(46rNX z0(uQ2mCf{cj<)NhEEGL!X{>nTq{|HIuX4`ZDNv5oJ}@;eDqgv)Q}ljq!kx1rF!zq* zB{tCpx;)JhdmFf+hv0=n&Yllt)=5NhYhwb>1iJV!>;rX3br z#=Oo>Iu

E4|EVZEx-`z1Q9F4Sxy>se{hf^G~+$vX06SthDOrNA@RtJI&3y$ytO1 zuNr>pywbQzy`h=SW-D;jUqM@VzOShs*;0a&zKTmiYpb`MQYzMZKjCy~$Wov+UG@&k z?n@p*cY^!<7B5H5FAFJNCQU_z_(kpq8sG}HO} zx;ba6KyN>m)`u#%2F!Be8L?Z9+3iU1Qhl5j+vD-mUapH@zO^w|cEz>#D&*#943!8nY-(JWVDSqtF;(TqeK!Yxtc9=jK()&& z(r;I>c?~wxW=`wyM=^@Mr>_lp8>ZY7S0ZB>1_0%MCDWgEbpMO}KwX^nPi`HBaCAp{cp<{uvPQ{(-D21L8@xyLCvo{+6~y-vkNA-lFQU{QlE4{@3~&(g z!R)x>;MPr7*6%M_!oJ?0SwGR^5Z79_oOf08022FtZZUnvmx`*Ub>dZOHYsBZ$H2P0 z10pS%Tp>3lyp6F!btE{ev7X4GxLp#w9(r^xoh1tuNt8oK46WxE+z;}Vn9*`l3pNom zJFOAY8qX{e8Ds zBU=2WuMOu`R4!qb%0Gl>nVI_*F5D`sjA9OZ_?VX`W-&nZ3`f9Q4zi$AUPIlBXN4Z|LU0$*^_(kzS#5p(O~g^=g|m#w!o3-fi(7F;g8_!Lpg z?JdSa`&B@pZOgK#D!rU-T^qC22%+EfJBJ4i72zf4B`F~rL97$+w|(DHkDD$s!7kd{ z1m5nB^nG=UWYlxtFsi(zQ|qck61$eh<=H@2OdwK zi*9p$aR(KqBIGT4WMxo}6>5E9KoZSja^sn^vxUI}sdwqg#tH&TyPw%N+wP0#C+qrx zMJ1Z_iZnal)uOx>pTaiij;QezLqsqxr3DVgL*)W@i@*;1oZWJoAE&0?ueqM?7}*wT z>3++7LM}4DpuFU^)TP@O<1Mbg-%J_^>-s37|B5AfbaPj6YYa9&8mc0Jr>sC>J%)Ss z3eW^K-;=&qzugn8?BY}xdt5zn63xBBA!hoz&Ej=waua&QT57m6=%663(dQnejyFx=XU_`vCTQLh=ZN>b#wqZMi7EC+ z7B`E`Vv!sHY0;~o@XgTt{)V<6bW~@imSBLM9e3Ju!!#r$tEFJzO$MqFy%4=2tFLE; zqI7JlnKaZh#Gm!wugVI^6ZSg$RJXA|pd#&qi{wQ7ZrNEUD-i(borI^{$0uZL`i6&nsIl>((M7cRI}OCx{n=uB)1c z7y|SdO25gVrv65QJrnPb?OmI-oA{aRZytHWEqxP)aju6)i7}ni=DXi2rKKbapYG_7 zYqIYfb48Km$Y>k|?@11Am2CwKPR~p#={erno$kr2)V$Alh4J1-!+EyV52$3s!vaur zv*4~?te?&bQ5%wnp;QnPHyYW1Hq9n0?`VIeH;!pBS%0-IPyxGkFps)`R$BEI>rAE5 zJh~hmU*POzoormbW#c!hQY+i2R>5hYH4|02ZC0cO*I7P)qkvsljb7#LJ= z+NXuKMCV{*rXj+9Ewd$T>nm8NJ!i~Tq#qw%F5GBjmu72letp&WrF6*V()IcC@oFr$ zAA)YKv>4~mM&3x)(bl>aiCe)uZzer5dqWh1Y)J=X#SOIKY>w$x_5D2yy#=&WW^LG| zFSb#JRW{8DXW|~eiB8Dr8O;__dB)b8Y*&wLgrOEkTu8yPVm7Jd4`zafhG;`W4%z0G z@v)Mm2y3kJ19C;ogLHyjYbtR*y@iC$8aGEX;?;)dVtwFXhatGq#h0A6Hx8^QFm#uO zzwEwaT#gN+C|pMf9EtmNg&&#msPCl`vbT15o_HL6nICkIaTBT&r*OI5uy}WksR6eT zO8aanR?DTZbGSZwb0ztvdBZx)5l2ddQA`n1m2A){d9EkCo$(y=Ea3!@UTIxbf^3X> zr%;e4hpFaK>s8c|;j4>={DciXk@pEpFr08@+ZH=+w3$~3Cyl-7RQ><4_ugSqblJOT zBO(F zPdPJZ=1VhYX3pGuf6sI0FCMC^cI~R#d#|NXAWTve7B{4n7BTMa`YzO z*)i7~^pAUOI>~J!&17;|qq+<6?XS#qBT*gVjeR|vEWARL@yF`^h%8(T*`H3x{v~HeY_+V>vy)5 zdK%7!R-6^CLT`MIyWNpz?wFRlK4usp?2~NgEj3H~ZK~4!&{*qI&(-t-c~Qoxsc}GX z!5nW%;3XoV4m2*5?TQm&Ii)ijhkH$LR!5uH(H-SyIg-lXpHiY}c=C|jj@-&?wYONO zINE_tI1m7vu&X#BOiNoA8W13`3TSB*N>Yq}yD_Rm{#~ZvzS)GL*LP;7s>m|XEH~#$ zL3pNoPwly;4x^DE3AiO%x(RyfyYJq-aDil4yqlQ1UF3xJR33a&y{te<&{hQG!(aYP z$Y8fMDcxJE2ZLzRCy@odp=<DTNv_OJF3wa;Or@y`E;D!-6!c928J^E*spcTF7S)#WY&z-r%DP zuXN){pQ4`r0^%cIa+}y8>5OdxL(q1G)aPRLNp0rKDAH)_#y)Lq+E0qvWZfL>bG+x= z7Ox$~?ZpBGv_j%OuWk;Iz|($dcMkmBs&Ac{XP69Q-E2J}j!h#1Z_Wuu@si;8Zo+S( z(op-Y<6uK$x?6txwEfpgzIlz^_$C?F`zo5-FF{1-e}@PGNDkZ)bt5OpV0@zW=jx%a zt=EI`O|H9$%_OotX#&&L7Kbau#K*Ln8vNF0*Og-mg#8n-VZlrXX3RuDQ>0+^f9%E$ zN2b{Jc!D@UM{>bsBkQbWJ8adn4BEeWv0Q@#G*A9GQcxusR`QP|23fq+Om7ZI+#iw8 zyfNDRb;?KA-Aks8*>)uJIVeyU>|mhkr>ky!zR`1gRM^Pi7cAbxw~_vid;M+m*dN+}YvZ^(jo9w5}!C|WuC z*p2DUI@^_L4t~7a(YzPGhM`Yf`N~`SdlSK`Fzt~87jmPIqm`0+8+yYMi;R;~7Tp87 zG%-;zxh#;`e#)7d@(u!X`0bjJ$H;IfPchwV?Am-;C5Goc#Vej~VuA0^1uqDa^WP0t zG7+o-1w~j*SjXH7FSt zvYhXf8#2{Z<8<6c-hy}FbqNmupWk=61AB*{>I9tGIN)PAU@iX$osSMxZrUOWkyye@9SVdlxi1!y$qZ zx9nmc&K}E+>{pVo5&l+j-N2(RvG3@)4Ts`gkx0`)>gfbL;P@|^~!a->SX`3Tu&0-nmiz}l*$IBzWV;QO%-_Sp|m0wqS8)Jk;t0lJ=7VgYW8eXn-L6fFV} zs|Dtf<1^R~xO8;lEbEKcYpC+WNL8J=Eov@&7)FrkyM}|H=DJ^AT=|5uSiAX1%r0e0 z>q4B$#@uVeLRUw{p<{>m(DvcimoBO zv6Fw;TM=4!`r|9DcDpUl1T8bm)O*>5S59|}YGz7s*Sy{Sn*31Hp6-VG4K*)PeJ$7- zE{^kh0~f~mWfPRqXyLA_m6q-TCpi7IB_&$zW;+E`;^@e1DA`LPxl`IojrpaT2|2n( zu~+wou$PFhgG7KU?cW1l2cp6!g2eaj`^M0n&Vn_UNBm(X)zw$YcDL-gp0*bsK4}e! zd3OBr{n-M!CAmscZ5Aszi(#}PyOGD3*Y2)Ek#E-mcqq|nm)^@p^nG+B72b`RzJThaFUd| zj^{0$n+at@R`_f>&X-cJ9T#cO^Qbf*j~4j7cTAtXKWs5!%GgeT*wS!2ihOOo5*-RX z-)_tTzmhnyx)#m$toerM#{KQOI3Oy4$nl715mQx_-Z_qD>9*JSy_=aqz3f@ht2VR@ zZxWQ=E}pwBMcJ8QmvBcxchqs8)trS}#<}&q(s8aQG*_HP7y?a*elAkr1Qmthr!!^0|XyIUX z%rech!F&VHTLD7At|vp9Lm#7^y%Ucgz6dK^xpVCOl!ThKwle7`?!vRexs%}2e3+NV zRu;h86-k)Gjf0Q&fbskTba-|B2WXi62MD?V$X~==#{mtzd13h0Kv#E^1q~ z9Wqa;G(}H8%6;S==QkVD<6%?p?v#|w2|3g$3{u^lM1-!pXtm&!O8nL~fJRDtcu{$d zY0_bjUqI=7J#U8S4awo2v3~pzn*=H0L-yx!iZ|OxpKx^Dp#LHT^_yed+%qk9n)xfdfVZJl7 ztl{(EFjVFKG+n<`UW26z&*LAU3+i2-{53NfgkFw@YEvtxqy>dMuC%WPuv-QrmGt1C z_8%Z#fXf?zeOpzHE6#VSC{97nza5rxs2P~@6XHu;I`jHfvO;`t8;2!-ntLfFr@!B+ zM3b2CegxX+m2EoK_*0q)U4pZ}r#N}sWr@p07m%&|$ksr5v=2#}?FkS~Sq?$>QzmDSE#*o*skO3 z>97O?AU~^S;>Zdkz~v>hFzopl+3(`~un>S*d~=F4jj9wP#kuFT2`#cw)1TVdS_e4u zfm-fIWA-lRo0k)CXJ@{B*LK8vkOmYiI%x~K`~p;IXEtXhPDTZ|_(E@p17tN;ZW43g z((U_NfH#D`LPIBiiGr)FvO+@ zkk=gK`fB#4pkvu|O9eTO!Vcm*@z*U8Ileya>JK17k1jlUS|lwP-hZY43#F%%;~;TM z1or+keqc!+;i}VxjE1t4n1jAd2(`TQioOiLJTJg>*0x-C&O^(c(F#KE&ah1@0o|Sx zD?0ba<2;v+Gd)f$Ce_yd#AP?PIWlGg%?s~d*v#Fc`wXH`nW5dI{}4EE`@X{~0)mKy zljAjUd^mA{qOZx9x@8eaETs5Ujy&mhQjU( z@>{i5pluVcTqQ0Ky`!PKyH*{qnmA#RZ?trA&sM`=ki+VhedJxvHNzaM@j_Xh_JGyC zyhaXq0QztXj8}AqGVF7S${o?AzcX7`h9^*Q_U>My@;wYDKT#4>{3O}y3niatZ+f1A zM%?`TrmrG_F??<&yr_{8m|slxgIkMZRk|K7O41zU+B_K-TCY90{*_!Xfwk%A!z=uo z>UQnD^ZQacqb-QpY)(DoYv3U>;Q@IT8g4~B^Y$M<)m&@o<5u+6Wvt@5z3;;VSLQ`J6{oYCK8{6vrQ8C?@(K6RaPao;W|9qiw$b5FtOX`?9 z?JzdC@hm|e-Pha~#!1<-L${WKHgi0N>wjgH2mJY@>^=KL@-y4x_tiQ}{CaxzZia1d z!wt}~pOxubU^vm0SXI%vfYU{fa|cSKU5ek3**?47;`r!GEu<@C;iTy3QS8sejE*R9u>(^XAcUD_~f;)i) zMxOY;V~~}d2`GvB90o{;P1IjRf4hy7Z63U@;G5T;7XT*8CBwFpVhT96FMCt6ta5<` z60`*~64k*rKdl}_IH(Uv;YQE~7X23xFNO0TB~G;U)~_~pSi(rZn^)z zJMp($*8Rg<0=qW+sUL{dQZ1k#R)6Zp2{VLS?6|P^HzeB@u*|t>51$Qk0j&J`G$pP8 zJvM)v77C0e-UkpqhW7HWX@LY!ZR%>6jJ!(D@r6^t%J2Wd)_gkvoFv zXVL`BS0?-z8KDfqP)-okE>lVpjJ3g8Ii3k%h0!=3UPDyL0oZBl7ubS2@BfHw)E_3Z zD1n-0w4v@t>xx`n5o-7hop1j2k$dhVJociWf*2V9hRwGdS3zGdpR{98r}*$k1u z9TcrWK>c!nwmpdrAoI)DI;OZ=cDTzu3>vt|5X}BchtJicEDz-8(_$TJeKtrM7XzgR zJotv?igyr>R`KVvSKJ%T=@{jJRLFClT8uh4Z>9nOzDk==)Y8B~Zh~RMi&m0&#p9Zf zA72RQ4J{i|mp>)fTi3jLiJJb}7g}|&El`fd%g`9oVvRPE=4vVxYD@Gj1(5PLuUr;s z;GHhE0_>Bn)K459D>79-mZ+K99fPMTFe9rl8^H&$$w4B% z`OOpgTYG4?<_Y_%oQ*Wz)S}M!Q*BF?PeeX^at?5$Cd_7Q3!VN>QF{gjPaM~iF>~zP z+&ap#cWgIXcel*lIYe0dZCO`}5aOT_2<&-h=&^?6JuO_>f%U8LNYY6nc<~YVg99NA z@dULk@&lAF{(gvq;0H$uoT(;`LZLlb7w4Qmp=_il5@6xg6B);drk&>lTs;?eTyvdu zrS`}?2J7uywa;Fefu8Mi5gg$a3i= zEG!N>`x3gVe((c?=h{Z?M}DoVpr0>=jbxb^5GXf}1vZZ7)QrIGci4sqZgaGrP5zZ* z3}ze0F1@LqLUm$a<6%siOEf3QDQur!=4ijXX3TL(Tl+D8F3a^xkoO&(;1ANRy09ig zTnP3SE+-S*2H|K-3FQB})!fiZcP6pr^f#8~kFG2sWeF-zR~0lZ>fGausvbP-la+kG zHbZNq`H_SIzrqGS;w?j#;=+)8yyi#|U$CKl$|f+lbHhkL|zM1HjlNzU8nq zZ(Ch$E3hDdCH^IW^B73r=$r}^xx?|*4p*&i&EjinhVJ@)`c&~fwYZlZ%YK^IzlKl7 zXXdVO@F+*FHrwzrF;K5rC+EAAIt0MG-I;^RG;fu<+Ko*I-N$vbWC@|~7fVIV7qjl% zY>XB8`YEd9;7nHoz19`uU@sv(I;zsi3%FYLuR&wvqEcmth}FBGs_H7^rnKEt#9GXMn=NIfwew3xhiyO(|Z>Y-fbq|cpQ(@2x4d`J1J z>M4;$2i&^vO^J6RJ3Z#oQq{XvV%-IKx)I(kr!(Oxj=G6@EWkq2ZBUz)b2;8_r=^a-M za#_r0pG!>2%1+2HR0G6qtnE)_;)NPBolb`hnGT(c6%iEMHb~q*QPx~HQo~FVc#dWx zuz;}$boCRs(p>aaz&f6_s~wa}SD~8bfqZEn)S%8`5#D61F^URy|7P4pr7&45K3*~1 z$U=CC)5W_IkP6asqQOT?3 zOGtlu!wF+~%gn=Q#B4q zOAQz)S<;eb0)1v9&M|7A*DqemwS9ClG9%|odOn+X`^3|7W$WKYW*OjiB886UH-yW~ zGl2z>&6W8;#=&7&;e!E$>zcaMGG61u{qs5aK8BQw+w;yfY( zkhpc&lE%<2>ku^_ZmA08egCRt_w@9&Udo%BG2=dTd3gu&jiT@J^dNb|LsOau-$Y2% z)%G#d%m97ovj~v3d+GMFvFVV`gkwhqS254r3tZB{CFnj&&kl4lLZ@XZ?&1}*oF*{Fl)uMNr5hrogt-FaUxc!Rb%jhL-5kD3FWX0h%gze4~k`xY%7xYcPiRsB1cfP}~ zt7DPUL?WiCrL}RpQff_}wNZ}7!}x~B-n}y8!p{%ajNy$;_ykAiICsO9ncdeH>IcI}-z?f~Yx@80s-1!Syn0 ztc5OTu3US1t!9tvQINisx%bXF^EcGjdrUa?uITIaJd@XDB}tLK`%427s>dtMZsXOT zv&1nX5pbit+vZlPqbE+wbtwsmt9-?OnOaJv_N*4$7v9WwW|!YmI@i4%7G6B_#ULXp=~10od;&+jZN@e7kns*Nz-7)fy>!U0>FSDM}er*yHnEj_9ea zmC2$`7Ch@80>o0|#gKd!JP-ji$#Ce7y+AkwWR%Ki&!w*&9b5FStUXIQCJ2axA^@9g0qGwDMQsXL(uC-JMfHXjXi9r5&xB+K6U&`nkGAGg+8$Qi4 z=DK4;8Xa);vf`guzdI0I@wt>=_HIch%jM7GqC4u~Y`}Z&gMtvN;czJ6j@}0%=jeVi zfPu7!rX)T00b&h;Fu+xw0=Ay~-DxKPO0yaqIQb0^uwVg9>~v|8qB_+dTr5qH9s3ME zJ~u-T-0oHRpB>#lxR&zooCn-#6$`|WvT+^7oVCSI926>$2OPci`8Hut0; z$d@qcywVszMPdn^LIk2sA>74x8(>;h%ln+pG+SaMTM$x-V&G-%8M~mhA&y=yc0Nv2 z`B=veuk*AxhH~=&vpqrM7QuVgTF&Z9I{&CdSick-NYpKpEBJNta45xX5Pja?Pi#Q| z`0r_0$MR2%kCAO~v=b$c_xvgLiKBJ;2qq8W`Sw;}5@&S%x$VHidGKJc$}ht_I@2}; z3yBp+xAc(UjuCHeC%k@2#|K-x40<0)0G1C*qps14McHvW=4x)p>(s68Jm0!c54LpM2rl z=MU!j#d`67U1Ii!q48f2f&r|xNo$LB1*?MT4lk}uyH$8V8vJOz*d7ddiB`iqWl{Ib zEA(#kNR4`C)%Yf(ia{>@f4%xIl`ddm(x5{AeCn1h+GsX!y|R|~W%V7I5tq(xKrObd z@nD<$^IEKSS}>&ims#UhOatoqHi4sb0;fEGUjC@E?%jTY{iW+WXKtRmuNEAb+fTVX z_8Ey2wr+(nujiZN_PnRr2VcKT8QOBSsX^8A?crOcxo;rqbuN-tzc_)2Y&cJJ%V%pU zxDs0^ga>ESr7>i)*x`y)Esb(Z?!BYcfw?G>FGn^E^q&`s*~8kS9Xj0KZkQ@Z18@t$ zDjR+beZK#e%Io1qN`&?61i7RL(3z4Vx(?l)I)C{k&5g&$+@CT8FDe9RKO9)%L^RXm z&O=_kvwV;;Q`r|f(=wVUROBSgU=%U7Jw$sj$~a;d0Vq!^8DC3zPur zVC9(meVCF;fQFl{%V7{oHfu15*%v&Uzhw+?D3tZ8yh?lg>oyv){wWWoIrGx;2auln%n%pfu~eia6VO zz~HUPTDAq2t}2J6x(Kz%v?lbqVqSY>$Gv`O7px~6>~G4VbY53`-=V;QE>G#Qmo7JI zvkwjI=PQfWYC+J}8~Ef;Ir&2;RgWGD`aG$hta)oyv>Fv;evA9vB5zKj*Sc60_n47{ z5XZgM_a0M-SEkr39D^hNGC`~nZK3PvSZw(vbwcvsiziRRgttq$ir^JYSqlYyy@1U5 z{P5yUk)D3DZkg7>M;raCjWlc4OqJkp3s`ugl+_yB^tr*;mLkJ)lanv<8YAV6wU-47>kAcN)K;4`x!ZO<#&;Wo%}C{{>v&?f>l+p zPl-vYJeaTZV{P7?#beLvecoz6#PSM%pc`yG=kc|a{V{!1LJ5bR$*ptP0?pN&h-9$W zIzUp*V7|z^>z%Of7g~sD(uT0FdDqf5rp|&RK1W4gv5c;m?4P`l1*e zg_YG)ee20Q2eAixN~zdTmM(kgjYVyD{6%;W@wL2*q7ktW!oQZ%$WfN1ra99@;<)lo z$oIK#-peOdaa&fCC3L(%In~rBq*z)W6Ld{!FiIB%bCPedS%HfX`uhg6}t zW~<7Q8Qy6@=`JjJdG)qbXD+_GrGcdefp5&abWUYu*t|(H`EsO4q6Q)~;k`GAZ(Bg! ziRgR#53*~@e=a};JYfc%^0iZlr8sCoX@~07g3ec$5kmkbm3JLa#~Y0v1dxA1eArkl z^9u==FwHah-%VE@wGX!#NBipyzR-pAI8{TU`=iY+={^`ib`O=+VKnnkRSg9s~#Y%-0N;HlEr(;F*4nuMm%e#|a zH>M@YI_7`8<@0uuGl-+10W~x|f7FFaGq!wiyM!|PpeHbv?EWXhibRUYg;cqk3BAsM z0xf63tm?7GOnNS8^F6e0o(^46jq!n-W#iUw}-v7KUW2 z8_HpZ7cO(0)IVwR(L3d$AFIiS$+iUVYfsoj3O>I%JmK`S<I@IdiQI3%uj~U-tQSgsefh&N#Yjp3UCo(c4h1bUwq5D+uW5D|Ax2xjlg&H zxXII%px1FG6O+LS!i6*=K4@)GppCFmXrJcL?thmpX7Qx5-p9rmLBUO~%T1Ks7PF8LW39Y3owzJDg=OSgKR z@2tA2^&Dzb4IQFBkW-IK&INPfJ{}SUOh&wP3i24gL!}se$f~^`b<eB;yqk1AP6^oyngHmB4nWf~20AeXV_ZJocxECg3r5LMwcT zjI`uH9S!$G`t$pCe)i{A?ZE!@ zTz}A?-?j6r=lWfL{@l*r`a|KOg;ge|d8uesq|ZquGJa~uI|_$s9G;0eO;{z*B&K=+ zNs{knPyFTUjq`!XX z-+CthTq*89e>vG+IDy%SbaQ5&4PR?b%9K1$z6u3a&RtzqAk@AqPEQoM0b2W=J5Qgy z1Dk-(=BDtv^77vAN8o|iBmZcaLjs7|OxV}x)R;=?$QjDz@U|e1L6_x^ zBLv3yB1Rb532hNNvsX(Hvs>pC+ppadNYcLK`X175@TeeX`_X(*1=ah?DjhE(;ttH{ zmD{B3d^URhd;X?1Mt|dNV6ojpDUIxKWX5e`x?d1dB!N<7c_IC{NSdHwD_uIWgept^ z0V?@Oq_HM%JOwu!Bgsq!4S_2IKc?HydzD-AP+lE!%T3`Cx}qv`k!t9PpFLLc$p-P93pML7Q4y;=ze!*V^W^o*1_MP>vo3aQo zrWUrFmiX=Fv0lAw5^y55MM|PuLG~V%HuKq%hM@3a(;f`vnR}PQXeOrR`7?8?_nnK} ziYo)t>QH&9%U-8-jE7A2M3C3M;KDFg ze$}3r>pr{Jc^mhPxRiuwMd={r)yX9*i%}U1LJZu2ac*61ZD85iLHq%snlLapIoaY& zZ8_O45Pgb8%ZX@B8S*-{KZcElgo8rH;C^6l(T`ZQvLwNg@1meWgU^`MtXLh{M&<{Ls505 z)tdd@AVdylJNxlfHY5*G;&`ZBOdS0FeC%dr>^VXE37s7;J96=mn3`Ggfp@C)@w$7v{fvCRmI# z+Ot9$0GDrPo|RO=jXKBWko;Ol^ZI5_mwe~lS#;JZz!v+TYWm%y|K$w;e<(Xp=v#;* z#bPE>2aPmQs4koS!(YaO=G{oEm=jTeCX*?N*9tF<=r?M$#)%*>+2sGj{-@_B7NcCH zgMWb7Q4Rp$Y6jL-TcN7nC4J`at7u;L=YEWYHesc8y%^t>*0gmNj*@@q=wXX9jwPS3ilgSC7& zu%*K&$I}qHwIC7nzC{pft}lX9-0HRk6V|yh`K9>#jF*$=J!<$?_zpniJ5HeUWK`5N z7eQnRFelJa3-%7>ymc!$!n)FBFtvKjOB$zDue=f`S9eotf8YGJr~rPTdNf$ot;qyQ zPZGsh&Wl7hsuAvu$RW7b{2OyzB`sTH?H=EHPDb72(r~q8^WD*P+hp(s!C-TAUko5G zWQ}EkMJmIOb0HToj|=Zt*M@E$3cA+3;{Oz*^y(>?kCjXPjQ7XWpR?(-+OjU-%xA4B zCvY9J9qq8^wXv;RqFeb1x>qC(bpr2IzspiecUHPA;r`Rl592k8+q~*@hHXS1yQ?*e``wrCRXX`;Mf?i|>Oy$VHo6%@!Q*BJm!j^x)&e`I=XCy)E ziEk4ABhA;=9%3r*Kc@X?1DlpkXbDLJEePRC(H--sRus=Vf|taPYi!XOl=piaOZ&i| zf2O$dta_VnS5=XnT+x;4ZV$T6r*#*|wyet>TGg!EIDMkw96s~FXtIDkJFttrJSuM{ z7_+{j_Y#^^&cR~-3ZB!+aa8sh5Gcz7oF{r@z2^^*$Q&Mu7<=3B1C(pnPw5YU>jpF9 zZxroZV67a6mjPA>V5Fl#A_P?g7BC9g$ma;OAppJuk;wn~2rIBm&%Oa~uHiC`6r z|66|tc+P)h5r1#)9RFW2cVD4Su^~H{nOU|d2*<=p(!lUT_|}CQYKl#|3UwNAEyuzw z5q!}w*)rfJ&9r|Dx(Gk_;p517alP%FkQGhq#N#8&7XhEhV!I!^z4Uvc=yAzqV}#a^ zWv$k8S80XUBp}a5UH?B9B;-nkrvX&rY$>9y_wcH?T<4@kHe}T<=H>z z(=^spEkTT2nG&=(xG+3*$+=GX9GAe>#G1rE`U1E^5;Xv-3LgS^(wsdo1gjsDVOjnT zC;-@30N6w8#~#TdBcKOMj(`s^x=%JmU_H)qC%Mx=)5={Wm zJocOru*`?5{-`ivp+NsCK)UmE6TZo8M*#nBm<*Tb?ymNI*UIYVp04aG!T4_#DohEj zS-DxB>R|n#UgQT@YitzM9wQxnjdT{Bk#1+6|AMG+piG))#lP3m29w9#x|B`tN-QMN z!j*9%Z{ZZ^z7`a{M!qGgbxN|tK_sy4d+?J+M|vs0OFXBp>r*n5?Kc1b$7gszNXH@S zG#r*kkRF&c#7O1Yjj|d@cAG4z&I=?M2o}Ak*6eM`SfmD3pl6zurJLNIr74EmeF~!i zpK;`yzP*3t%Uhj@SbVRY=K&BRk-$KnSu55HV676YhoJZS&JZ*P;g%(R1*>U&FQg>5 zN=U8wU15S@SLSRrq9_DKRk`D%K>iJF-%%eGo2=R4N_FV{tdkf-sF!hBLa*f$ITMS} z8d?T(fjc%2W$d-cg?wmpgB^i)&Ww1-rK?O!Za{ohxv}LWWM)5V)aG4gc0=nFLj05l zWl8tSk^U(F;XsBr%3!oQ5FxyB1C{RXxxR)j_6Ou528!(TJ-Q+QO=tc~#Ps^c14x?R=>>OZB}1`9{9 z5JWIpW7uGv^cyo=!t#bTr!Z=6vT&rr`IB*rmV_+yONqiep;`)dN_POGsfQ0IJq#L6TF!U?pR>)kt_jIb;t3kEm}xJiAV_?~h3DbY zuUCJ#-d^Zt@0Huhs2OVTmPEPR#gmh{BIwcrx`TrP+0F$g(jX^5QHDlBc!%~UVP|p1 zlINbbAFDhYTC{wQH81<($F0Q>pG&iput?s1{zQHYn4aK{6UD&V?|>_DCXK>qM0CBi zR0Ca|OOYiIz&Lbx@gKdtZoPt#yzmJ0U4N^08^rMz2V1D#Tw6e}^#Gijp3+w6NnDFb z>K&a+2hShldt`QR`(?k<wRUA|=8NvQ?AKUAX#&s!HiYoX)-Upt;I% zi!Un=C~~%XeuC76;3PQC*Yz>L*>Qbm^x@ah5|OE;oT0im2FjME_&{_%!q-6zf)7F8 z)j`M9zk&|_(bE5K{dUwsq80@fEH?}daS8~E^iq<@_QB0RKu5w^j6XmhSi(t(d7#I? z3rHxx9)fohblmX50O?PMGZ^yjADX4#_wC4Xp>GqpAb{ccDJ73+#ZI&1!t`mab?Uwh z60|rM7BjWJ$3G3W%sE^9@*9Vkh@L!4t_btKg8+D|sRRyj+k{Q1V7)LuKq2zJnq##? zMVdKpZ(o?LG%5sL-UyYV6maiQWoqh#OLPte(jd_oxA7fPAlwBgTV6Z>8y76X1n6w*9fkoEkFD)cqNxT}OuQVEk&d7>= zKfFt)3I5sQ#{X_{KhPvU&f@|Q1H1r)t@kz-+z28fIqOxbA@Sh^RZA@3g&?iD5OYdu zKX1()1)ZA*ayEox95BeKI#-X2VSJ1CfQlhjVisy;tbGQj2h$pH@}vB3AOHIv|3{5S z4iN&g>8xcuYw}#_bN}M5MJ*SHxqO)~fy^rB1&NMDk}y8(!`|U7fWm7%d18{pP)JX8 zhd_&S2;5y)PI{yBH2I1#|Fj5n+O{g`G@41&^xQ?rW86f%79x&p87IxI^2WXl9FHISe><@E18f5SOYOy6>n8Aqw zWnI$|0P!s}m6R~`4@IcJDgg803UskINbm<}EIFT)YS~ZegSI4HUwH!EH2;Z_r6_|o z-~9|UAP&S&fF16OKvc4Y)?dtps}5K0TZ!nWEMt4I-l4)$PXv-K7|OEmol+CLwDuI( zTELU|l*C2g#!E-Q@AF}NUI>B(16?Amp>;v*|t3P`nx&f3+m9DW(uok;YNqC7nx=7sLnkBiMuFoTq|$PIw_c0;IYo3-UpHfAj!cX z2KYJjOw$bg_tgoWDN>G4!L;_f^|X?l^C>&}wQ+>Oud|>)5P7VJHXl#VAG_`FqxnP2 zzRAN`O{7a|R-ex3_KH4aQP=U&`;Ko(_W;-R)87IF`V@#<3%F%w^6KnLtB&01ESZlD z=#Os&X1gls1*=Q@RN=?hDDSC*js8C7{5LS}-w%Jlmk>=F(8V6oIYeC^P=Pr(7z_!) zSp!+ap=N9Bi4TC2^EjI>>vx{U(VYd7p*CPM?OK7WZ1pKe{u90VE1dpsG&c4)np2y{ z_?G>v7!dEiy zQn)zQu%*57c5a$>z(kc@Nl8k>c&90gW-qlx#pjt>MBC|sT(>ifY#EY^Cz09o)~OKs z-uaSM#9}{CR~5^(&K|o&Fg?mBDV>`NP><6nUL8x37@x#GHzyCIVuyzM`WDfNP)Oa@ zRl0oiwsb9!q$Ri3o0VuHTbL9KjA9Ys9mj4Uj@Oah#x3&zMC3WP-*hndE9mu4V5{4a zV#Dvj*1wIYzdBM7K!4-uGels?Mg{nXM+o_nkm|tN8huB_5NF|1D_d z4Z!TomWCl7(1&qWw?LV%V=>lZU5){qrxMz?=xj2{1XN%Y9*Uy&U#0+1_i9TQ-jBcu zML)y6m>r32i&lhL3%A@EI{5N%`O`BHTaS$W3$kFJcJ>TH-{JF9JAghnpvn=2Falhu zK_D?5^^;6TWBoHXlLH>i$oZ3 zg$|4)3qUddKW>e}N^9P2t1#f93tSD)u5BAAO2d3elBmi$DN=Wy0kBKt%w(&G^1jWk z1q3;+&;+-I*=Y4XK5Q*LJ5ijb;nq{~!Xd7GnyESElKze0Q>Q7aRW1P9sDI59Py%$; zDjXHY2&1dP~7L}xnT3b3UcfWi&s z<$(ez(I}F3{xj4j1z`Z-{bvUD%wV`#=hli8R)1l!~{_w(`gwt(sO~3X}4{K`KRc^hjB_{7#qJ$3L z7WsY4Xl8Dh@JD@KmVofUb#b{&w=g6WBt5@^ioZfM49m|Dsz=3@F1b$Uc}Mb0xYE3i zX^+zytpbenLO(3XV)iz# z4e`4jeM9y8h0(f=MYvk><)+?))D}Pd_Zf$&=oIXfWlhnCZx&)>hqcbd#}7OQr%8ho z%);xwZVt~ce3eibyNQ@R&r!BtD(3a6gM!yjRDO*rh~Pbw2bFOFT&)^UnyRdiI1~E%=O*X8;buLDVudV=Xv;Od1_;Q zbuD5pPlcK`@J30TU#Yoi@`yAsF zv2V6n$sFnOP75bWoiBx1C)Z0kYFAuH6eIzc_Hh%bYc5?Er&x-K2D`{f`ugjTkh}n2SL&vvZ;qxV z-`CqEcOJa0GSRV8orINtTMM(+OXAKe)IfwHd8Fz!3a`$gSle8f5BY?uzF)nYp!a26 zPiE@*=iXCFMW5WF)@=(rC}Xvi962PV@bXnNI+j+F1($9%jigo-9wr3!d*3<1sb4!h zehL0BN&qaPtkB5g6(}^J zwo_**x$idJC!T5QOGJ)!9Ro2P+m?{?9CLtp@}n&NxZtj{;QHz&j@16AAfw@3CGv=c zW3!WUa$8nAu4OyY0|z56}sh} z{Q%kYrUK@zte?{US3cjr;?skrN#CHmfP>`_i$Bjgb1cINI)3_}HPOE{eE*-Jga1(r zzeEziiP7`=-dWJRe|<6eF zHk8EoMP`>VfA`;q_~9%UXpmu9vMGI*+LBu$h_dyIACy9qKY+<=GmZ&)WfQ3LbKRqG zwU?i4q_T@AP+aw#5QU= z=H61$lHZ&78E2%#P`x9td6`_0?$%1wU8I}XyoFM`m9nwD(^p1aHKj`blEd|si8W`% z0WCJp{o8iL4X{l6t_FW6lT7UeKyQyP=9%e&lb+Ptc-E9tdVsbAv=8&Ts--wrbF9 z+@>>eYXAsM1R5~;ZPqjdg+aDv-cw=I0UO2_-i!3m<`=3UxE8Zy$5fuL2IVWv-)5tq z8?{c(UEz_NP}@HKN>rJ7HKR3Y%>knU+iQ8@5Z36GRGjqON+e0#Y0p_Xba-GwHNB*Y zqe)nkv}ub=2cucYt3d=4gZPC{=?r2AcEgs3Z;uIjTmZ-N5P$ltU(%Bwcg$&>=P%!d8IVPIBjK;G*y zYl*>q>&`4`;a7+p=fb@8lmtsF%kN@3%pnwdT0cNVywAADR*WAVJgS=F9B@yvN^?k> zJFk;+qZ{vy%8f1l^@M_`0UmG;qO)NYiM?cwSd*Lr1&03J^?+;-;GzhSZm!J3h<+A` zHLnJc4}t=<7~{)CYRd$$dTrZmy08EC0syfFwrE?-C-s4``jV)9J2|q0f6-6*R~9dz z>3^F-x(!_tOqs#ngppF@L4gK;_l5m6#Q9~%$RZZNoWRK533nw}F2pjJJ@8kH-2T+i zkZBXGb_!-95}YHL06GbTytewl*NOMlX=O|GhMs^>!f)VSW0e%-w^MUfO>U}P-ST5b zpcse0y2otl=GD*>6csz%jxY2@fN_j7glL1U*HeM>%a?%?vL4JNDN1%;i*&LYmq57&LuUs@f1lw&t+qe}5wkjoU3uOKp0R1}FVMM~Mis&emT>?~tI2^8_yIa$3Fy7N!}f1CqW}8@ z(f|Dh{~bl>Wpz|_YFifvE8=VrkPc;WdB`U0wN$Qsq0uC_q9$4d5L%ul~g^NwwiI^gF1&o~||3Y^^1x({MhFLpH;1vWIg>bBq; zQ~gH5`*=|_1eq)~Ove&?pN9oS-cC($V188T=}UB=u{RGKv_c{scKyvup&U<*$5bKG z-Fqm&)Qyt3Z&!!Fz4E8zqriMxu$hUFeozp>VW@wR@e))d!CgV*91a}0+(~R zU0>&vSqV3T+o4OxO)P@Wue%Y*(fO?#(ow2xg&Hhv30&B!%A|)b&uXSMFT#kl!chd!(jKLIl?PCa_8wA!eK^PfOetpS^U zJYJP*zi~5>@9tF0H9k_+Vn_6a$9Y?hch7Mg)}rPBf;1DwOQRR{j)eesTElXN8SaA1 zVVqd=x9WL|e3S?KHt*~!$i4Eo*KdI@`Wxg`k@NlATCEXRqBx(Sk}tI#ccNO-`pRSq zU$m{Ukl)o7t6 zZX2bwZ;uilR-6|g^M9@Cwg0$Y<%86Ih5u5`@Znmu#CwrtT{RBSk|*3UcEF=AS!kem z%-`#-Wr@PWBHL+_p7(7X0kw*Z2JC{693C~zZ$2TMa)%IG{=?@JQ+odS?_c6)1oS4$ zMBJ{OJzR0`G<#>^^_h99eILbrsJHApuHbJL)8l#I=zxWg!EdH}rHjNU*bv{a{mE3Z09m=Z4~971@qdcVbXvqiXV>BSVfKQNx1>8vr*B!?pl!Wj2P_ z>2ZO02%Srs&IL>m@jZj=dCmiFn1pOs0w6Bt83oSioJ?}(FHjgWDh$AnJJKR(0tX#X zs<1h(tz#;{-RZM_U@c)S~<2T3@xP2Nh zCJo0r$wp5H9eHx?Zrr;a42%Rtb(sakep6srgMXRtghnO9+4o^n>7>Yp(7VJ3VJW22 zh=%LOv=t`y2yV^6Oe_WKAQ6!lUyMD2gRK5mE+Md#$3S)gjFt)A%(6tSbvZDu|6_sr zU;Qt}RbW&&pPGtolRW^mY-$!p;2{(A|#pwTIh58>o z$biRGaBtQE%2Imb>NB4l9inscloy)IaZMQeB||kP<)K1+C^{Z3c)!RxyrSq<;^>nc zF-~QY0-1;(JUdGUjsfzz4Low5Rghz40YE1>V^u$2Yw5<#7hO>O`=Zj&sH>OxKtzxRCh|DY=8HC9r--pDr| zwBqexTwJ2)*}IRLI=D9cmK>ooDf`4q(GYL>{bKJ54?QS5d^QU&5WY_;jcRw3;b)>( zh8Kxs@0fqsaekgwS^s=c%m4X>o#w5AtH@EMn+7@y50!053s6#qhwPcfN;5bW11x=c z`Z&Cs3>3CkDxuTxwx6|pcV6U8>zYEE>{FZfyCk5u@NML<*mPjW%H=6l@1`9@QBP(% z-lfVJ5LIG57cpq6cFDP%MW-^(-|{3q&THqFvA?JZ@`NkLENYAjlvtEk+isVp#g_i8 z?KI}LC~cP=TGkg?dYAa`oap>Ekh_>)*t<};Wp&Kp#uq4^gz(Cjh zu4@1=&~*-eWB^PNS^tfA`{%CeKQ~2mXhWitONl8s@C$_EA~ejRW~4-HTxjM&gMmX{ zyjk8FCx@iCuA~e;t}s;N^;K22sTn$xDh|#~OKHLK z(VZ5Gm|g*?#(cf}MBTzbFhES0oU&=`hLXPuarxWc@&6xmOzSStr@r&cxlaHLinEIy z1FpS>Ee4swhRr(MW`y91DOxcXz6D)ppG#z%LNG4SAUG-QQU{x5qWg%VFtZe3Es@ z71gz862`G+YMxU?(6?##OzeS8j7b^iOJjx{s=EWK+{u1aL9G}Sn4#+iv_0Jdm?<)9 znF>FQpo-6j&F@K}mbQRWfm`n=Jpna{0gMW%uTl26U!W8sY6U{bL2{#p$^gA*HH2yf zS$6@*bS>1bBQV%{jfRP!!`lHPEL2M@+3FWayOVzBJBsM_50?XzkdQ+ZHyzj}{^3;( z)AN=vKUV=b7}&!^ApiE7;?N7|){_}eVJx`U;oz+l0pQIr=G7>R-dRV}4Waa#TGv zonm^moDSmrx^^1mGk;P`ntHj6Y`BmvIUm`?SzS|2*iPpvGbphX?$~hVv9*7zl7{PXb=SGg&8gol6s1CK&$s;S4 z24T!nGQ%}2JSV{5_~s>VE?}Ak1a&HnhLqF#Of0mrY2~;QO5RZEl3nPraKhNnR8zkXN)48?rh&d z)6WpA+XG9{7L%c(BZ`LFgRtR>rEqmaMd{a9O1?>$53pK_sjO-&fJ_Odovs2@C@nUR zyuZlCwLT|8ywrHP{Bx+O9KVxOSZ(rgzm|(~+J#Raw=Lx{FkTA&9bry1Z9_3@K%9us z>$I{8j1W1weJgjIDlIx2lU&`kT_j~fZ1cTfxU~JqCtJfSiw#E0Mv74Hq6DcN!*o`1 zwgQR_OT1@p*j_hyG7?vHS-k^E(N5^T7!z#9vh>vGIGdmj5e4XeeSwe-<&J*O4hu+> z2FwxHXm2#-q4~(Jg!+rJCcn z*Zwh3U#`pL{QfqV!72-k-!pv63}CMH~ci!@}tw7)&w_REXo--)Fc9L4YUhk6#6_=p|eyY6;^#q%Vb(lQ0^+b zAu$sV_~o>r_+~ax6_uD4AnM%GI0&LwR#jh7-7qHlDqeJ~MDDwd)Cmy(O_hbBbD({; zDOo&gBat6Q^nIyeycne3&W8?By!#}L7JThof&F>ci-=36?z{8qem}-SZi2DEE(U@6 zt<1GYHJ=6SLvu*EH|{nn3ZhlTTnz5*mhujt&hJ}S(t zWiCsY+u$tL40H4AlE0noI#196A)5b3{f`Pvi0Vg{BQ&T11?M6vd`~}T!^!ib9w~7z zm{S6m%y3$ZlrlOBWD>EknE)tnWEF$YXRgE1kER)R0csg1YoO%1`zZ+TbJZdR4oSaSh<9B2rBR9Uw1Uwty3N zBR^c2UVQ@Xyac)blYziU8vXM*XG%&Zb&pU1KCI1|@duAX+)+3)RI_#kuKEtGL_--- zH0+9yewKQa^dfyfXGW&wsAyYNp_F_XnSkhO5=C5?7)OPy4EUvDcV_F*R6k?Vu-JO} z_QQxpIF$>o*RqZ|-gJ)q!n7&1NZ+BK+^-~0W+|EQhc97!+A_skW(ttSCr2HGpgyPr1!MXbh%gC zO^nEaWGy}dt*ei3K33*dMH#04VoM1xC@%d*PLn^^rZW>~W>-3ena*VVaU~EPgJt$g z2)OQ)r_oBhV##^PsS|6$5j2Eq5d#MPfmL*KgA0|hy+f!o6t3! z7I3aD#G?++MT5shXTj8qz~m#_kqR(+L>7V5b>27c>-z$C)=3PNE0i9`iJPaN;s$sD zhtEOxtp3lv0OUvo`uFeP`X!)UTgaueTtlF6#(ujhwN5HGszz@$1%G~V7Q0|}wD-QP zL0YG5n5s@|*yCJuKnfv;n|>Q>Izvu#dp~XXAo@pjgvurLgu;*VvtL5InjFV@ zI@zReW8{G-ugBDut8ZPvMq}oL3Gi8JrjE6lc8oUPOAmQ=V7dR(0;rDJvUfXy(wOBt znS1d>87{Ej#qd#}X`H_6)0mur-L{0}$L4XKmrhA}#vGJ3_PED)F5q6psB$)C6i;u~ z`)CQJUoNOGYQr?Bf;$=C=(7SxqVf!<)GMk`01M{>i>pV)6j1dn%idvq> z$6KkqlV*9Snc;U?RuNTS{!{n9(}ULD%r|LBy?JOkrj0TWIQ_DecyC#c?>k9GDs1WA zc&VFk`-Pbb2)`1GmkvI$_zm>fbSaf?#ZnGrfROnNy@&S>6}Fb0>M^J!8`0R-%Y%y+ z(2bFK`O%Gb&Hv?Ih%E{0ggul+m4j($3`MijXW;-PY!L*jlpIp}k8rmCfF%CyO8C9v zD<_+?*iHc`ClBonbS=(tp3D#2@ST4+5d7cr5Mw*^`OcIc2|;$GT=84&6gok6vdwi$ zdr(r5>>EG(Q2j=~FOwX@l7vTb>+66auoa;i2);`5-U)xiKB?#$W}?hJ9T#Tp>n(3J z%P8{2GDrQ4D9AQagT?czpdM8YvRMuoQ$+xB;;%DUy80gg@c+a^0@dx1sc6!5vMJ#w zoE%m{y*L`17iJ?m95ZiqIqj;Pz|}*n|M&N7K3oika-Ps>7u(uho9KQYl#JZV+wXAK zHr0*5@2T9h%%;l-{|RV0f)RI+Fd@e5TljEAQWAEs+}!(k^+~vgDs#*bco@TrvW2rC z`;{=qk7t&-7jGO*ww0WJ=DRDKA|(98M8#(H$!}ZTqHlL^@_-n(03?KQkgny2ZWlU7 zz6Lb=&3xyIipVdDEhY`CMx{k+Yksa=RE)pNAfkuu)9|`}^)#8Ausgc2+KfF-mayTv z;%MQQa`*LTW>QLDHH+nFpw_9iETAuH4>*y!FfT~PFGAwHOQ&W!uxBbVeA|3Ee$=Ie zg|%lUdE4-B#rQva@)Pb+uD)EYD)z|$6bPJb4YCwdWWO3Vw294u(+_1#h@~1?l2OKA z3{1Z?RH#xUJM?wFhmCQ#oocmV0UZ~FE=GiM-w6f0)vcji=(ESoK1ppgus@&BbqPyY z3f6e<86kNX=fGG)ZpB%79FSg;j{`+6PgCN}q2@Uq64vk!vc# z7{mI8L3zs)hh_Im(Iq=^M<@vDJqTobb)>jIGG)=Q!u(YKp#S_;AKm&CDYp~4Fa?G# z>N5gs$pRWZjPQb-v|NORRhIB7#Jb+D9DU!|_$5v0g!)@+H$-1yAu;Ckl(6Jg!O~_7 zH(9uSPA*A9jH0m7#2huyqmQcgc4cnPJ>k*QbVJ9Z&)ht+ndhYEn9(uF7uj&C4z7fI z@%|&>A#cIUJ9J?u?unwnX9V7@jPp|p)?erOouB{SrJ|x?1b&Aw8WAH@Z#^hPTm|Ba zEZb1*4PqZ1swSEQg5>q+6X@Hizxlr!;w&@&!138;a6ls3H05hG~hC1U7 zeQ8QRNmwagSi_H$ER~s=C|JHty64HWi?4B&UXru-D0p}@^c$6z2#%zmLKqVPV`9uX zgfuZ}wmJsk>0Z2cG?A36*#9DASI21d_DS}aM#ZZcSy}-s!JgChVR%}mhA$q7bHtVd zHFg_>iHx?uJR0`y=WiWaFQ1MqYnpiKnx^;=bm}|@Pq42W!w!ICD3*ZR0(A3>$*6_A z9K!mXK~mV`u?oM6ne?#-T}kiJ??d`d=x^&;fcB<=1r7KD0p}wbIXJ)nqZIXntAb&x zpTa@!z0*E}JC|u7kVxun!XA)T+as%cIWwDAi+Oqp#Q1In9?FPg((v2 z>McAmTtrX77MnhwZW2YEB!@hxDQf~P zb(HQFa$I4M;K%GoP6Yqa8u*TAsryBcP-7{uL_OlUXrj;HdA#7B z{9J(b#oNNq;6X3AB;*L0m?1c|g9W)yAjk(0Y{LhYxLRh(f0Ru|zdtZT7W5 zyVMVkYNGGIIZWH;h7|^VbGA0ifxrIX{47^hCeI`1lnYuJ zIG!Gpw+PDgOMs=)WM`~}lYtvQot=%Rn>4HCE$|kf$k;vbyHa5p{t83CBj4k5C{1M?y|{z+atlc=M_>O=Nv5Q7B{{vkh<2H@>tQzb@<5p8GcHl;gklN3z+P=@b>=>s3=QfGmocNp-)cpR3F?7@ z2VUH&Q>e7*FVPwCwS;(IS5Q8a+ul$hb?>8>*W;cZ-D}6(t_l9&VFL=!|Bt3G{@&|u z`VX%=&Oc-3nLxX29Q3hyW`t@%cb?9r{GzvCzQFRY#-4O*zZD)o>U(?pVZpJ-ne(tP zoIoe1;eEtstiCd;R?Jn;epW~jd7*h2bFraeWR&Pc80{mXqaa*B!xp8s#0NkCCCCew z-z}%5(@J7|ZibzmZN4xY=Xqf#hM_%3#x&Vp)sR~|`opO2BqmUFSf(>JuG4lZOy}PF zG)K!jaH#V4pFv1G={dc-iQWbUM&K*xgJR#B#z}0#gVPACyla2CkFxVBZ>Qq=2z`bW zPipE-UMHdduW+Ga=fJCE8)ooA2JV|YLD1XJ$`26R!(x3G$>7bq&G~O9I}$T z*?WRfMYs}yG;TAKT69TbE|7jcX{b1|;X6my8+x*P^i3i6-b9e;z9)FmHwK`-A5v9_ zur|o%{1zKox58-_zM1KpL_(LhjQ9tVs`_W#>m55hLq{6PdYQL02Sk9R_b?oqEbiKY z;d5{!V_KkP;OPBEp@o_#is9I{%#%c$vu?%7$K+T}o)^lB`r+rGGc|Zb=@;Nw_r(C7 zc&@A$dITbhwW7~GflmJ-h7j*>IzfUzq4-Y8uOuk@XFkl38+FxsU!DV-?oD2d z@y|eC2kcm(IB7y$#A3E13BHw~G-I@@(N8zc;Araq#$Q&?`*lE#DN<7DrPB!eu53H- zw%`#|b$l--410{u>xAJ#7}^TAdem84yChs_Ew)oxN_g_}OxyXpQ)rbR;k-3J4X60T zt*qWP4pO z>;shPxYahW<^4%oUEIhX~ z8Wcg=pylmLURbo^CkYZC9&uXZ-40?!%Bm&SbxQj_c-G!8ZFJrbaMIJfwPZEuKug0u zca4BlB+fYm*e2~nAk1^K6ArY;ZeDrGayC4@n)d$%U7JFRp7p-W; zf50ka+xdt(`#9iz0mS{wbc&vN^i7H^3H`01kzgpBitR~M*>$gWpiB+&NW{uIOFn7= zzM@$46b8Q?Ptiw6R*>WK9HCR3ien_@Y1iqdk>{(iO}NA-F0=f)_+O+UmYwll4qy{@GFE)0?=ysSnvbpMnA(cGxG%r%x|BNGLB7LV^n2% zb9ndhtfIlxa{o}QH!xv5BMwU9BM;*-Xvi_kPUx9WB{GV4 zi)YPUtE$X6dtz-Ql;Y%n3Oyro;qUmoT6tp?zbE?m>fZ<<$Q1{;5dFeDeHL)kVs!U? ziV)5t2PvMsaIR2{lRwa2q<1WK1Pg)PTJu``Sgw8UDMt$A8KqDIU9!lcwD6dKRHVn0$#8^n;m9Y$^(xH5J#{mJeR6c&0TF z_wm&3R7SVRV+F^qt}gIf`T(FTGV~Xv$O8C*1Aii)y9kp13uZ-YQa&3`>lz3^;FGf0 z8mUm+7Puu%vpRw*pELEt!Zyv{hm`DAd{?*W#eI-w5a76(4FGUV)`F{FQ#GZK6O zvEeipXCJ=VB$7eC-&h~wUU8?!I6DzKUc+JnBB2SWfm z;o|-xasptvlhV%jF0HAyu;RDlnYbf~8>dc2sW58lYp{re_Uit^7XCB(aCp$s2HpBW z_>JpA#`DTm3AHc^*LVDA=pi47g8W-WLl!1wlYJd=Y%@*^fQ8y#P1B$g-U?+9*fnG= z(o|U=rIg4R^L8bg_&qQUM49`${|KWqDKAR9wqUri_O2Wb7H21FJ#(H-H+@v-eQaS+ z#%OkPEtoOoTWQ>>V+6Ka_e@P>P8VE4XCv9@Ll7n{HHROCR1Kn4<@c7~57eIbU2Lil zE9^a`9?q(ZOHxsJqWVc+r_*yJ>}J1}P~^?e|K2)e0xcsiBJPv@3F+EE1I3Q8%6IUQ ziz%p0IY_{4xAXM&%x}I=+%mjGx^&OTG*L^NmV^|4h~ZQxyB(m@2t{MMa(sBY7(tw4d(9q2Lf(#^y>DquMStgP%ed(J!e^BfMax0B zQ;eKEq~}6$(cK8)1v9hlIF)RkG$O;=LsbB~9rXtjm>jlRMk&vazI@#N=kMun96YPj z;vWI3YzSk5OeG)#Xjf`+R<^7Nbo4v^-lpeP>e^bf70mAL%~TyjmGVBopLtM#9D-tS z0?ud+-#`+Yi+XV!-uko4EKxJuZ2Fe`z--|fc2hTXX)N=``^9Pd(J_wyEpq*r!B_48 zCdYiTue{herWkTH>q`c*W2q7J8ZAbZM^hTeI~8$p;@R*XsST8^oD~@`jc?ujQ15 zAkX%j?bBT+k&ES84+07vi``?rH@pX5{YD^Cud|3gx@F`U zW3lS*P+C2xNZC2FdYP1=*FGmO(jXRyPoKTZeSLJ(ZtRoHFyGbW;S2tdXCa+T z#XiB7F*X{e5Ro^v(goY2ZTE`wO<$ItD@6%+Kyi&wfSEdym_OA1BPq7(D8qG`dUueP z8%B3+27v!zKYmgbEkjFq!=3Eeg5s)}^1j?7a_->u;1m1!?-%vhYvbfZV){TgMC1I< z%C|lUX?X7lD+TyyM&m-fg zF1&&QYWikYUSeN+53L z%1wT_i?`eOWyx*_FQd-8ZCYlHfzb8N@=4VR1VGyz2WH?C{fD8OJ|n0%V{cRUGW_f> zcOYzuB6A#M_@cDOx4!aDyT@U;HyuQM`IGl_th*#ePT(;ql>xOv=T!PSk}f+3))(a^ zmsk!=sL0>iIUQPX*VQTL2)-PM$xJcT5ON|VBcH{`d%KU>#q0AN_^KRR$u7SUvzFB` z?rQz{b5+{P%8E`n&jlbwT3A3rBF;SWJV;e!!tjkbjP7#B_Qgx+zTb5kP@2K<~@s zBjm?viLV7HmD*}${hWH(ccUz#A!+P2!+P<_qe4@>qgUBVEyzaf*avtF|E1bEXC*krydYCn*UMr0Vj@W~pbx3LACtgZ% zE^rKZbK2F{(UZq7FE*U%OXkT>UemreFl@l$*AyE=@*(WV!xSa`LPO6FexAn32e;x} z8_ucrvB{SC^_N+Xp0_C&v=f_u`OH@#vp`-FR~2SU(M2`uYM>V3{LjZLFku^_nJM-f zP9p66uHqqz5`HZ`E&#Ig>8)l#H$~zUFbMe#l2zIE6p5nGOe>at#Y{WlxVN|$zNS5x zC@)ZqYqE~H_)yA~qrMy6Fmg?!tW0B3QB&8cnw6HTYHas;T)`VkFcFhOFc2z23;jmM zv67NK{}zcTOtKIoxzWBLMUeg6Vw-db(aw8^Mc41DcB+X=C(X@~9bJ-@Z+TFWbmybT z!TZ#WU!dxP_EaP{av-fgGf6Vmwc}0`|L8rbRJKgE%pS0Y_7{+z?$iCV$$SF)XOB3K z`v6L5b*dD4p?Yh0fi2=RljwHp0!;p{ru`+E*1F-gPw z&`wEg3<=Km$&SpN=TJ?Tc6ucoE^S`M)G~0+oMA51gAtpc%SA1A%TL3IDPzmhfUH;( za%kQkdveoRKe2n{vI2xN!KMDhi|0ql`Pnl+et&+hOy*FjtWwvxkf%z}5`>HV`srs&N2-=Gf?=30-Dx9)3HoUEkU!mP`9l4Rr6|K%kxf}%+hd89DH&7Hc_lpKe-^jf{42vPf0XZtfi^i?C6zoN#I2}j2y%*VdpP9VYb9HoCrSWma+B{|P z>5^Yo7jOgJ8=t%vD*;Qh^ign+@wOjHi)h0}V!xDbju-i@l46N28FweA5`c z*gw4+(}1S;gaq!DUj0V>M*9GyduXYXBT1!9RDrmS79k--$I%NnMT1I`t%zm8o;Doa z#2|T9$eY515UwM)Qf{nOEB_FwgNH2%T^w6}a%Uy1Iqa0CL7z>RMiOhF5qJ}Dll^CH z#UD_#lHSeu+@x`K20J?y5bK9D5Vek0rFHocHHg?}F&HvofQi7IuH+9A^nTi0-88;! zYb=&LE2w<~BrMFHJRtfo7@0HlcaZi!UgJNf$^Nqt*MO6%$;{9#W&(1fB|s^w4i%|T zR0BAE&eoqbE5)&J$+K#Isn64!mEs4xoCNpP`ECLH1=I92^b~y*dYWpAV>{=#e!f0K zfhe7X{|M@5E&@znRE)X@S#&^vU@_v=ACWhllR$fV9}sc^ET+bFk_%uNmx%=y5Adge zfO^~_1W2zrwhv_MoQ0DAuv}t0Lrcf5iIoCONdaKR3e3~^Mf{Oxb4?QkY)vpa;K0<{ z2DRHcItA>CWQo7n^cj4kLjLuM>-rMUx+ z_Rm%=&g*gduJDNYsX(P|ec31(OcDg#6AT>@aQxW` zEn&a6kE@CnH(HT15$>GG&4pd%2Au#Em%q;)WT8weU6;wk& zQ@JAA1FXKh-J&`#oK(C#eCO=;<2027ZOw$s;PJ@)0{he`laE2MH&>Z}@QVK(hQ{s0 z((uF%3tlf4Q2P za_6~i?kf}*7@`rx%^wcxL$1IhV|_k8@Nu=Lc~fbyK5~vs+f(*T*JQo4vZ>Q4zNo?Lmcn!B~eysV{Et%7~;`b5^rh4K2?T-fSdFH?S4lABzD{lA} zlgFm0@wL9kg}(~m%<`_4vd|QEAw0-l>}d@fqVOGZe9+V3?-o6y-={CTURt#I1(JT? zdpu4XNeL6J3n@v_B=kLNNhuGQv7ez*>jNARB}mSe`yIV!^>=l18gG7RJeZ?&0Q||c zL6FGz%!AHv7piC&r0|TTcDxREsQ1e_1*5_!say~Us2Rq4b*fFI58$bLig+IDT%ICP zZrfcAW0C|}l@Q$yFP9DV-r+(>5|CvDvnYqdgX$x}lD6HgF(CQw=9>BE!on+lAl8sN zId6Nu{>xBo*K+ULy-cAiZ;S!srzuO)Fzqe!bT2{Of|wHCAc;et*wEsN-Xc}|80q9+ z_sAX6cm6$6PV%Of*$e&#JYV<&Ulu}5ROYZw)#y1WSnpeHy9i)xwEI)jhNMgEz!LdM z9t2nv^{zW9_KuTk@oJ6C_D@>cqNQ!Ju8*CgptXsQSPD=HEm%i}gqiCIs_-QgMa1M{ z4;>Hy!+h)q*^+E;$oBGy{Mz`GcdOCK`5w7DLvqKhx z$0_dc(g%jkohBg;0F?wctr^-}@%4=_xhHNH`>-5ROA6(C5oU&uy^6n7He^(kZ>5to zmULS)claBz^f=i7C{98`5l&-tF>+6kNh=Qe(uJe<)^Kj(^Azy2@~b60%>4S5wl!}3 z7mMR>6kpXBp7Su1c*ah~0QC`z`)Fw2hFT)|T<;MTiQi#kCEK( zn>sRjry_3ptexr;j^DXB{KW(&(=V&4cYi10PBweAn*+(Yc##*V3&wTpK(x1c_cg@L5eLtI4Pp!(|5ml zIkxXb*EO-XXRQ#k`8j%{ML}dMkE%IPjnf6A2@w1NfU8ITvu|askwWA3P@ar6G%YW7HP)oR!jV*GXPv zZX%fUgfJMT;Y=uPt(z$I2RP`tIrHMgn;g6Mjm&C?>GlWLz*~lNC7(N7EXxEXvN4rW&>7W=u~wZ2z9+ap!^tE9}m`muT-@ zx#np-p!z&UCejn}m?_Ves$}JBJBTY^*3w?W+gajkf!mUiv+XM-{@eJIC-}REanDAy zVlw@tA1AD@JdNTp>K2J>e!#_&puYf8pEw2qBb$v;Ed#W?CN*ln=tAnAApj%O6S&o2Te(T=mmgKwsUld(>e)4$fz91&gC^_ zc9%+EZfAlW<4Qo-#AH8eFfOcJ(>2I4FuN3V_)u5x>pbBu^Ea(ah-YL;Ahqw=!SuVM z{F!}zhj2U>rvaCHS?6sLUzw@3!cz;AF8tqL=Ji{@f4&A?CzSnuW%(_zW*Vo#!BBRb} zypGZrNwD%0e_*oCsc&!<*ji1^o1H20vx6Mg+YNVvr1HmKDy>0pSjb`84v!o5LCNA@ z&}2Y{SDv`wA|(BW@jL2+87eqGjBLMlW-9XqcM33)+dJdf~fj z#{(?ng>>TKE{42$-EdV-mg0NRJ>i&d=-dzX<|Z7CPTpp+r`01Y?YL= zt2J(hQbZHGLujv?nY}nu8)z*qcSZn_-q1ju>U>sn|HI0g#|ro$c0ixrtchCc^#X{D zjbhZ=;WKx5`&rnlWnhIRo=g5gfs!+6Pwr~W$<>6-3w#kxO1(?wM)i%`Xeq9#)hD2u z%@!L`Q$Jf1uIg8z+esa6U{8A=F3j_#pqn{hBKB&2Es6W(wn8tK*$!Uc18II!Y*7}Sd4{0Wh$281Tw?LpKCUwhm42MvsG9J+F5*ZjLdodgtvc?3D+_sQ zjd>Hyus2DS-XV*FX?&(YXI1B7iwq0v$QqkwPge&~lpF0xAQ*$fIQr_;`X*!?}LG01VPAOTP)IIL{qW zxftoOl0a8yVvgF?0kDT^>_Qe~&+(BUFwFX;7eLehu=EIP1kelMXU;9c4*hI0oEi_{ zA^*iMkwFT|30%AG#qN7rTEp2~p%BX*_SP#w?kr0~$JKYHMgjU8mo7Ukiw_Hr#rbz0 z!NMCjnQv5!!{iJ!qIgO_x+?nNCxWk_Tp%Da6qmAD5477S$eh>_?5DMbjV(nsn6 z_sQwMt>i;aQXO!j7SK{qo6@DIwIa1B46|Ym^+u4~!~DEI!8Tq#Y$)Qw;_3Mr`q|-q z>4k8LZH%$YvB#?c@0Q$kzNNuDW5=*|=qbEO;y@D_vS6Pm8l4yF%r6I>lL)TH!_us} zy%vrH_766#DALb<)4?o!3U0D-alP;~0<4Pfb*~6yVM6R~%i6%o5`8JhkloycMas<^ zwoD!xB_@CcEs)*?-Mfk1#TKG=4TVOm+V=y3Lg|7#l=A?M zd+`^jhYlcU-3o#FzHCLKn=nTT1HEhF&eMnmDvR0U!@7=aacZUqcmnMxgWJ6PDLglS z6GoAo64qmD4q^`#z^Jaip}Q=hb7lN}A!YgB)}*j@Y*W3kl))-8?}9TeChcrR&8jNH zP;gMEf@g2-qaMMy)|&+I3k_&B1>CI0nqIz5OEe4@Y*>vglpBOvq*vXGvJ?(@@h5-_ z?t63&m>hcR1R)ui0s%aaKc&G>tu7+vH<^ZjCivf!0=uwjJl!y25lxGlj-W?A1Fc|f z(T7R^CkteSI|4hV=(G5Z5jx+V=AG26zln?fW$cLIr(qW(oyijS+#M4(vN}&wTF=e& zk;NZEtVdL}|Ldfs&&dVEK^(N%P*V{SiB$SL+^{@*yqr^d46oAXrpu?tyYTrT>yi;T zJRiuS##Mb9K(G_i7gM67$D>3iGO}{6T@duaDEqEt4KV^6Ps16 zhJTP=5{j3lpM+8*fZgnW?Rh{AGfFgxB{DDeP;ZPh36M*h%bk+%(&`4=&OvPbw=eQ* zkUQ?&&x(anZo*U-7{3BdJY zm!Jji^M}rFzJ8s`7-Kk^zX}mXj%ZyWM-!_Tcafd*C-*;ASQ_U`AH?pzgQmLLQS_^s z!=&<5FNo-N_A8&_S8%vA=zUT<3F@kd>`aL+FlIMhf0Ka)Dn|HdV|~Wi6e=f{WK&7# zb&w}}nl&^z*v2>+Z*Ve?+e?{T{b(}p;lkj_I$Gg!y8CxxWqM<;`6FJvT75uiX^rHJ<^eOAEHYI-TkqM zeF4=wgq%eV`~tCKolBVzzYW7$(4kt~lT_~f>nP2n&8?NWJMY(h-=4fj6!GCWEc{sJ z;#>%Na!qp|7m;K-cnPz=(eu8%?X^5}*!n`Ig@f3}H< zfD8o`H!T*hkOnyi>59g4BkJl7gewnDD9eFdd{ph8+oncpr2WhnQ&rWYKj*F14#r2?KCjxu)L~Lco1uW3 zk3(-JD--X`PhS7}6?T@-*(mB6E%3(*8cyU6M=fN{R~!=q4w$%N4Ov2G%jD+@i}tw; zg?oxy_1$Ou$^_eXlcyS9nuM!zbtP!=FtuE189vDIhi(k5zmp?lW(S7R@b#U&pPIfV zWLPeOoffMBCrsYNCet+~n*(rZyJpGCwV4KvOUc?r2iJTr+)tLszF%(R-hAKNsi0|X zBk62J%dNCE%+g(2aK)BZFUf=u+n*8JzzOd!cxctlb*sc(yeaed-7{AuyQibn zK$r3S-k9E9F)4lP_mh~FAhw73e!#8+0ruNwBfw_a3q}#Fz$`0Xy7XtutK;kFgmO}h z-LEiJnv|W3?oVF3CE?g{>jz*qP)!m7+yRO) zgfMdb&6-h&G=!9$dMj*KiRGiO6qy{ zlrgRan}#rG*RAr}xbCRCPJ6d9I9~H4I`_r><+{=Bs7sEBPP2o;?ei`+!SdfJrmJU= zbH$FH;y)EW%_h`W&a9=FemgR0d?!#6=w(R_OSuD3HFC}>K`zjv@r^MY(4_YHK!7R& z2nf#?@svdMMq$WoRQ5<*(MH7;OSoB2*20@NpTVOqOs}_6TYnENc-c1m&bShOm z6ROL?oUtbCE*IT#M>@*(rv-3_r{AejPWkd7QRasAsp+g34$qj(o4-IBK^QPW;ERSq zCw?XOfz>sgrU%}3vm?$G)3+uJtSlck$+%;Ry=9AVqjv-6CXrTN8}t8-z4r`ja@*F1 zgP>8W(tA)4P@2+ff`uj`Sm;7jItW;3YDf^IHvs_wAt)e76={k{=m?1PF1_~zFa}fB zJ6-!-YwxvJ`S#gopYM8qoF8yqA>_&P%rfQ}_qfMB`ie0vu>G#cgJ=%FzPkI0#O_H_ zq;|hzfu(Fp5@+fI-)N>pvun?v+6PPU)oDrX3>)a4JHMN~wt@1-cMx5HFBpM>;capo z!AT6T_*!0<(W61VvotV$N~#1#iW)-_lE;G?)?HztwBTqkgVO zuM9mrNnc}0K271lP7eis6%y~blzge8#?srsA%-O`Nt#Jdpz_}AJf(o2i|g37{Ec0k zF4mq}#vjX56OnDPq^`Hlho4~*QXZ8uw5V+rxOZFk`9)vlB!h+kc?0;%O79cv;*@}xCt8;2IZv7a6zL6%~?^LbZ=g=#n-^;6feV}3NRJ?UY^w)KZ0k1LY zXsLYFnL3m?nTcXSav(5WCMA;vQI?hX2dm6ho2UDqP~+}2M?Bw}*VDFmVsK9AOd?rlkwojNUxr`z&t1i@ylE>imNt2MX=3IG zXR~1Rd$0CeSD5L$w9SO`bEG~VKFZ^Bzp4ch^4+(2Gq_fQS&%g*tNKkK#~de#@%+UhnhLC8UVY9^Y3nV>vhsT) z^Y*+?9@*N;5w8K@}7hJ01o zKi@lzaLL%mpt?uHOG0F-B!yR0TS1T>z z3vV}I##9VbEk(BGEE=NP1l`}ysqNnGOU?hvWx0L$^VQEw9biQ{$qAsm384*`gMNml z-D~TV1I4wudBs_pJvF6#?%0E2Gq*a!U?Iol%`@J5x#qhXeBIwB^#tRF9YMCh_yMeS zfO46nH>-hU@gvuJz#Fv|rWty|8N!&3zcbVQqAl58KVorhA)UWH^>%#EBL{cB1~*ul z=lXNPq8<{8?t$^6R9=z{ue%knu~^S6UQASr{A|Z~{3X7qUNrXIvqP>$$31oi!LAu5 zB1fF2FcGx<=&mH}0g8FvvdaA$HguPAx^%oXnx<#pXuz*R7lO8Z*W$)Mz-6#~#C|vZ z6LvV1c#nWx8^C9o;^ks0jJRnuP*|z3($kjRDgC-o&fWdsC5IU0@#=kI?k)Qli-nKm z7s%SHwHb^HrxUWKB|_k8DCI#oAF2FYl6tvYrQhv^vGY;FY6+SiW+u14*b5)X%T>8> z_2Uy@VoRpF0ze498EhqnWUe_tD68|e<$iN{Ukc{c8PRb|>liLm{_&3@{{1pXN*H+| zCwq=R5#P;D@3<2*koACwriN2I@y5i6gc)KkVb4$#(JCoa)2e8E?hVW7Mbq?wg~xjH zYlpZmos#EXOf{8i$$_6OWKSQ6eeL$-)89$Tpytes3Si*5542A(A{?_u92g54r$oDa z)#{pETr0vS5JUT(`Zem1#j6A#$as%IN+-_5JWS(Fgnkn|DDShb2>=YVExHF$8rqB- z*>I^aT$Ll~Yf1`yv)aIN@Xj(!F@^0q3*=sXE(Vz+XbY84I54v(NS=f?6i2?zz59q* zQ5&@?WiGkyqzcahZoh!9PeIM|>d$%&zs9wH4|eFbfZn<f%I?jIrvY)@;zQyn`oh*(Q1p(WLW zG8m4oCW_NA*YnRmNA}=>=;p8$t&}vjW*C{Pf{n`FPuGd=fNi;h6g*%%w-Y&(!K9X7E+W&1zdH})|8C8X)N8C)I z*k!rsT^f=_3+m)JSEdOj8>=dlJEjW!z2!FC=Vaz5uUcFz4a)&lyt>J69H2HLalj>t1`4p^Lab@4FziAXH{q-gt`xd{nI}eiXV+`< z5rX|18J;DSURF_=<|D7KAAh0rBSY3)UC7Lgs@;ueUV7BxSh-Q;&{xv4yOqfBMXS@u z(6777ao=?AwU+-NWi!-+9Rvm3so`mF0N0zDjz7h4-N&TNJHYDDxEJu+VvYd?%9+e% z_;Ju1h-q~Y>)NLjsa7a=ZJSSX_~_|)@wvmv$D17VMKuq)v|Nx;JpUq>hwo-rB*qRS zw1Oxb`q9pOM08eb?Y{rEtCaV2WUix|TUE-f8!?y2rVe}54jVJasvH}5#X zbeYm6N_TPE5noa6k0^m-;kjQ-7lqq8D7qw26#OWIAHzdAOAxA!%n|Du3(ql+%TDZS zk89<1ihIrXI$z(^QPuQ@Ea$xGR=rKiKoR4i#k{rXbb6cKnv2g}YQ?)fng>$~M$h;= zsOp}QX3zASbb~Y=ubT?cy|?G3L34R~)RUPUp_Hvq3KtX2Xz?sap^O>OE?+4vTUN(^ zHUTSKZIffSv{P`5E$Ony#@L$+QXuurE@_qW?rhejswE^;C6rZf7FGI?Sp}XQ zJ$I#K)0VjPrhnn(+ixZ9`9ttx&W`)`sZ3U1Fs>=6&0VJ|t&A+6OV5;~W~ z5Q-$brHxY4kZ%5IE}fXgZ)n)2u+MeDPP5#vQg^<7c>lGwvTAD;7uQR=(ei*rTBafK z-qnuRQCtiVI!YO7Xca~Gpt$^nac9Q<0h z`YRn}`bD5ZrbkKXYQlcsDhKH9>V3!tjS15xh?k&7sv~Lft=d9~e}N7Q8xib?q20CY zlrwl(W9Q{Ys}I!HSUH`<4+Hu1T&41QyfPU`mLu^zk88f~p<7k_^hxZx|Ol=O3y+|KH##H6b}PWpYVnG{)O_8q0+VjYxlm zhUI+TFZTCd`&UeR=6B4${&w*<+PHfmH%LQ`y`=Z`)!%|e`se%X2g^H(wB&B^J#b7F zk2;qndeqlL@GB?K7_IXpi5owp9D2oe2xUKbjga_--J_yUB>Y8RKYimlfrY0#8W%Y7 z4mU&mi=3QXa!1cxzwX9TQBf^z(#XgVqj)hOKPvz-MW@XH=0O-kNSHEiM8*okBDG23 z&(liY+$*{MfFVTSJ)f9SAweH8C6=`>I^wW{dlr9hk-9A2kqh5XI60pedjNP0H`u~Q zb)UOCWYb1qht_Ij9OQM}bF4dW3%+()sykYJuIjXsLl^_=;nDpJda+5>!m=!GM#%45 zs!7O8vh8J6o^rUIUB05Jl28~F=UOrXytUlMb@7EBFKf1gt11bf>L&;rJG+ND0RGV| zB>)`{Hl@F2!nxOKG2Ke(xM2|vK@uaYd!1Cw>$A3@uF~kf>NUwRVjAvE!PrSKz#qI0 zI4qHRHNEJ8`Fpi`qlf)!I#X=Md>1`*Z%4&cG#|KQU|70r8e0D7`MBcUpCEK-P5>Lm zk&K_CZ;=h8Pl zx7lKFP(`*F;7lnQhg$6bwMepU+gi!sxOjZaio(+cZ9s@TajCGUCXCP5p3}8j;qs~Q z=YmEm7j$9+Y4ilSMp&qoaRAC_R#NlAsGyXCoxgB0aUb{X{KAv^ftK|0bmgj2iA;|z z6y6jlUGgxnnr-Y$b8JPYU{18{ZV3qFcmA8o3x8m+A-8l_E# z$jLdSh~J)D^wFDWQEjWy@${5+v*>^Zf6o?_^Y^$8lk6pH?N`T^n&F+13UD^|CFca-t0r8qxFI(8Uj^kqfh5c z@-jlE0L#o=criP%T<;?ee?@LrB^~tgb(h#RG6=^S429eY1}1c9fBPlv9E*czypFuc zQ8f$7y(qz5*f*f~&B(_=_X5i~*-pL$=lcx3d}0+*C)bMG5hB_pPn+=YQxSP9{f$z! zpD!Yh@`=?NcHQmI&2Z$*YU7WYa8dbIJ|gonG0W=U;NasBh7*pvVeqE8h?Ne~W%2e& zWdSzMzG}}2m*UUnZgV{`qDn0@iTV9Lqa8Zi=h{-acAy!*JwN`PSASMaFq*g|@N2~L z2N#A(0enplGZ(mwA*#4w$YufXyg!c*{`*b#plUP5VZsdfm+zkYJ!lq|=u=$3+{Ckw z<;dW|wKwBN4eL?Dxv~WtS6Op8>3X__2LZk=B|-OleoZo&)GaPT`e9J zWxd_QnQTQrhl>iRQ!_|blChDyU9&nZEz3gmxYFQrZ=sswoppP^1Tzr%GFkS|#aIkl zq}vGYGrRow)Y7Rr0;^q664-E3*K4UW&m{LGr4>6@cZ}<0t#X@p+9_C%%TzR+yY|F8 znwU@#dh3?+p7kF`JTx9X1lzia;KZz%^&@r+O`u9-X-r!cMG-iFcE%t%Ds?5+HW@$k ze8sEtw~o(mU$LcnVCIk3a)IodjT|j${7Tv9W(r1hOkt)OaA@a`oC?$~SsNux8h=GvJxvXlap@J z*Br{16qxtxMx>(c=1(MXFgRA{4ZM7@+;0W{aDmx~@w00MpKG-iE|;og_tY~2gEvu_ z?yoc+iERk7;n;BND;Z5Y!-f#4B-`T0;hw57Y#qzCT}+thu|66dwzBok*hsEcun%r&2Ws+uko9(Ll;y9bm%T&;7)WsdU&FP@`6nDbM zF?BdtGT&?9g-b}oIW8gB&)b;LuS@=i{+2oU?Y4q}0J7eAr+|zbvD&tN{iqE9tqZuE zl{c|x*`L1guR33zk@;XDywHVqxTjEwDt=ojCfAK?zC0+a`3O{(awW)Cz|6<#n9su> zm%VnM5pqK7cc8~_C-`4{wQEdFvYUw=T_rl#BZ{{>LNdOAQv9-+LHJ zvEWXAY=-z=`J)g2d|?=%uhgGL%zZ|Kn((2-Tw1bqH`EtzP19d|h*+}(ET)|#b;w%Y zZ{U^Rj>I(OF8=a#Zc{8L$*lz~j~`v($360Z_vKu0N;=6AYA%q-D;=ON3Ym(EJzW>z z`&Z9{8Y81g0u}^hoh8a(y4iExnBLWgCKlC_lF%Nee(nmGMAL*$csEs&ewfCNI@e8; zL#cJw9$sQUTs0`eZh6G~i+e6TdtEF2f*y1B>@)jYIuP$2u#4LPU9KzSdr35H52oQv zwt+_5)cPqU>qvWPGiyh)2(wXD>9m1wuK3knv?Yl>*X)KaW9ueIoj`3PRpm3@?Cs%!8?pX_kq;`RR&TK~-E3o* z5M4ao@sS~@4nipTD?dGj0Hk#sw=H8@r_l-7q@c^a=4J$ekuQeVz3}h!4waZ{&fXSP z1NJcqWQx$vjM-zfZWy=SWB)<*-h*u2x`=ehoyjs}1X_|JPJKnuBg^`A%pCm|qiC_S zSv72nJh#>5I&tN$+plOEU$-KomOb^nbkDeTM~lfgY!_Xm8M-Jxz8FQY$As9(nTB%o{xX?ZJM^BTV%-l z%=SE!5z7hJL>(cWNLnW-v?}vUC%f7*`B;j-jXPEE5OUOoaWQ)Ac7}pBCq;v-Phlar ze-3q+#)Ye2cz%1vV7{{N@S}I}E_RGIjO(9n(!Xk4SG}$+#GOu1sk>hls{|yieZysP z2^S)7R{4Iv5wd+P_>sfac;%_^7fZ&IoRHe1Bv6$KB}JaXCx@d9+^-jqGR%VX%O9V7 z@QyR@_*({VZ^5|1009DYCbSVi!KdADhcvBom&`8V5;?{VLJ|q-rI{USt*Ry$Byy?2-*i|e*`f?7C~he*?zA#6DD{gzou7^*zQcEk` zF2s}Ur`wa7KP6x)wm+H(x2S13({BLvPauaOMlD9n4WlJNKwp%HSxU-ebt75;%9SSx zgv&QtupkmPdnNoqZo1@5|8goU6)p_i-B#o%J)o_XO~F`cJ+^ZAbZlrvwAsww*ef;h zoZ)hx447|=eLSRnJ?hq;Sx&kUlr}XTbJIAmmSX^Z6ew`N8lvc<5N=1Q1JUE-J>#DI zn{i2Mb4kY-^AjopK0^6453=Rp?6EOClQ0^y3r%UT=y{N3#k*v~{wAqPmA7HXwmE&6 zV+9Tkm(mAF(hq!fe{iN-glzV{aCo`<>$dF1B`(VFLv7Yx?+*gD6=aXlf)Q8cx!KR< zZw(UQX;0#`VTu6#FK3HF@sQ6_#7U;p>314B&JKMQa8Y{4b@8PaSwE`bb5mIN!_N@A z)fh?omHiTdD@QX%o#(@M&2^b`mvkTwAn3B2hOHZ~AxP)SG3zp8@K)kd4RxeN^TSvf z))m;Pv4SS_VSMgd=S=k7hS80ztQ8aE+^`a9*^CLlM%&Yc+-DDmoD!MZI>1TShuTXh zdV+zVh1}rEgqutc@<`|w1KF?E_@nO*)o}T%BJ|)BsZ)ubtB+D5+&Enp*@hEhn%(#N>T(CaN!BZ)4m(5lgRTb@Gsx zSm#&2nCuINp!z__c}xahLXBU>>{R|I_*zb+AY>0p>iHxoxykAYKRWrP^@oA+{`HE( z;rlGZiS(=5FbP^?l6qA$n;^=VT3B<&;pkgJZMN5keQM^csowACi}zGd%glU1L`_ZxxAN(7T_wpWA{BXPtz5T3B^XbtW;p%Oo z*QWI}IqDc99tn=WP)zF)mLLoI3A7+MYY&rxh`prTY=fL}zZaceC3}~)*{v`1&@r65 z+mll#GG(t=xNk(~{?4oX+w*#7&5xH7xtvY@8ubc8cLN<^PjytHZq%l^z*R#3324z@JQ$r&^M?=fHI>P?z({84Nnet* z_Y&P}A-rE?Do7i~hSDS5B^VNq;s)Sils$x)FdC$PgMWeCU@Kay|03`PV*bm=#h9ht9E_6$lSi7<4&bj3KE+#cF@&|E^50sAVGWs>BF*FrcU z`QyeG0I2FkXAYR8fsd9)7n1M20Hl^O<0y|pZ>D=P9+`1(;v0`7dR?F2Ij)bef>6c= zUIdO^si~?6T;RR&Hcm8VX>SVWNjiJ_tcUf&?=tuHdPkd|>MyEDXS-b8SHYylFgRpZ zrg-^`$kNA^QYI{_rla2DQ@Gg}MQhGAJhL9X)5 zN!RMrJ&`>%#}R)cx1rM+kmz8qK;Y=qjKp6WR$@wH3sxRA8XcW9oG~|a8MfCsX3?O= zt48 zQ+qe`Aw_`{p@bL(Zb>te7u^MyBbDZ1ZW`FVtWD2W+TlNcvh9fU%``q!N_p^=3~gN+ zL6L6I&vmug7g2!=go}gn*Qe;iDD~dWFD`Hi1B#X+_vK~2xVKj7|et(!~@YNr5ITGXRb2V|~`P_>;L9gD}D1Jpqy^k0DLdSvC&RnHf2dlp0>0K?b6b z)BYDUazBJNZ0iV(zuB^jyAife4Pqa2G5x}+X2|`}b)zXncm@ykMLBF#L5HM`<*ZS0 z$vPTUtiGR+}(%zzg)BVQ+XX)naZ&#oTLils4X&9d-g4lMP$5J+&1^704! zqomZh?b{C)ckepxmSz}(PBbVpd{*r;h~hhkcs5=z z_j(7*vW|0N5-<)L zEgbz+S)#EbY7%Aj*<07-Q^JwtL&cgEmvUO_a0&;FAbvx^ID2EZ75GnsiNRbsOXZ}5S49goIM*BY z@edlrPGWZ*_e(tN4bb8tHV!JW1pp}y7_AdVU+n2sHuL`7+Ks1IOVizJ24r~BzJ;d9 zbhp783f1``ePAaCI-rgihiol#r`EoqDRKwT4Dm08bx!f|1qqzhdD}WUMzLH7P>YSU zg0|oe0Du9msW**@d8y`Iq+rrFU6!_9;4L-c|K>}}QSXnk+0}~@^GuzPQ)E?i1N;CG zfR8;*3MM2sp^prFmGLx+B`fduY}a`?Oc&|bck6~eGL3JTelJtK82uUI{)8q33r&e) zPEFsa=!iaZ7C-mqi$!5k;u)^4C-$aGA!p;5=vuD$>qDRVd0r*?ldn@;2*%L}XKWNd zCRCFn(|C^bK)ZVAdMStQr~30~JJvmbT7U6fO20i_0AwfRv_D<2P#1eR0M#Z zcuC@jKrDOtzWz%eqxCF$nz(Q6`gR+VTff|X-!ZB>p8{ICeO<>8M?vk80(UT;Oyf+B z)C4^Nr`N@e*9A^i9;g|U%-)5cdHlXI0b5h3VY1JhqFF{Os%nLO8~O?22I>ujErHy_ z*la9$6&wpci?c|BUGGk4rG5n(pG&|DgeGAIDoL}n?>~GApv=;NSQ$gz{t4obA^zxn zHIM!3UaJbgF&xm-wXu*Q$IqL}(K#b)1Te4E_?8+C7=lY-$ChGiVn)Zibb z{Gq*6I`6_s(ImdhZxkhNLCBMhG*}Pn*z;jOx5m_)>Ob85oT8jUsYfAVyrwX1jjadW zAkO$NIn2;B$}P|zgA3hF<43ZfjPNtIO#YpA82WqWmF$8JQ%{*ow9h@uau-M}1wF%> zf%N+rjOlhZil3U}1>^KXpWq(sYOR!4YR+d3CqBJoAXYc!8vNbACV`&{7LJ7zv}PdQDfs!O{)qtWN!M07K< zx+=f^Wiu?~kZ-ZELSrQFYX>JUgx<%g5osV3`)a`S%JDo1GXx4Tjk5Zc@bb@T2xcr7sjLyb4^N$ps6iH4 zEvxGU7F?aUT08uLxt>R2JVmcR0ySt3D0w;9rdX+~G{jj}@D6d(aGD;ITmT&4x!j9Qsyq&q;yxGJ)lnDw=|I3Wl+Z5~Pb zPMk9?5>!?)&{+LqXXp)hYs)K7xWv`6rTNRX?H|?uLVs`#s&S~x6++Q_`!-`^#=78D zhRaKzcmttGg;F(XgSq$%ceP;`X`d%wOEh64aSjBPm`5~LeD-?>^9;Ei@^Ba8U2yHp zW7YWc%<^vv0|T_mW})?%P#6Qo88dRX5yM)e#+uHYT3qQ-Y}|F%ZqV&!5JrqmC{s8iZW@R zSv%89JB}#alQn$VjCk)8b+;2DoAZO|O?`$|AeGD**Re4!(^T-WF&-?$SJS(j@uLr{ zqso$WT zxCg@}F7GcRH%MJc`3LtHK`s$m8xS!-dO%|pCxmb+VHpNBV1yU+Gjc3fZI6~ms?K!^ zXh80aTs!5s&tPBgei8b%im|Sk)Xnu={M}}xK#f)|$-_wZo@lS4lwtjyn?+e{Z$GY_ zOZGvSu&xwKS}H#oxyq;1s*9t3A=N zg)OVAd+x|w$hf{j*SUf@fN=w9V;Gv3;zEj>8kalYSFUBlkg61uj&V`g z+IxnGLmMJz<#|L*ptlKCQ zjq2Mk*^2s&=yToslr}=ijsWCW!tV&5=$~SjPw#p6KR0?7~5X@-H8Q0|Y-c|YX0&Di; z`qTh5Yrx5dVVq-QZTEqG^lsBhiTK4cc)@~uo{X+1$0sM--$;i=5d_y*BOCWJGR^v| zKuW=XO|A<~$&J35o!ta$wc0nTu%eSy>-fT`|7oeabQk$wd&-{`NnpOo>%l7~C}^^G zl9PlXmXQ<)?}0))Tws|8$wA?psbH@S_AlPO<8sHz=i{T|)5lA1)mk_VkDZIocZQG* z@NTnmF*p}c-$|4AQ_#ns>{DEL-REU3RsQzvTgUL(ZKrtU<4t14^23_C%F*wBuruiA z9+WWn8P32coQNh{P694;?Q6RD=Jbtl_u{NC@5nb*9kfhC9WU_;pW^B>oYtfmx7^EJ z=K%YA5>#L63`NxLHHLq9DZ6anUCb-6#Pezk`A|LPRxVpN+X;{gf{TB6Q`1+x(D#+e zpva?W{6*@4tYcyxtU!ksXne3h&~@l07%yhpgR-wz3|GDFjbpJp+h3(@6Ln5A#rSUR zkw;ZNY|GlhKS6$2y8jZl{>*j>@eV0^$~a`QGPzaDxOGChPm#&6NY6a*K*b}ed=1GX#4(m^uFgSIi2D` zyNE{eUQ55BI%L^qOKjvQJWV|^@_=~vHRleovJvYq@4Q&jr!>Q!If={H#0AiG(b};4 zF-|2haKRz8B)WU@Ag&U1!J7|0?(}N4rK|1HUMa=s5BAsQsyg3_FqL0>mC0;#*_^}y zTgTynY|w$6Wg?otN6ARTD0QVXvhl%v4{NV*(UbDU!iua0U#?6g){6)?lbzl8k-;F= z>UwA^PT0FBV4)XL)k)#9hj(;g&ak0(cwNz_6MwGYE~BmW zM)J3)5rGa7k+0Et^edYqcWFi)(^b?+W!U$l&h(3HBLMyeu#vJy3M*T5yow2eGpP125uusZ$r&#u@0l zX__PG#z{t{Xr_h?dJ-&~qC*fb^LnkEmgASYjtaiX@F{GIUbIZof)Fo%LG|Olq4n+D z#vD*G>)D(6+ECJ6Ng&dz!t#8V zjfloLkMd=iNB0*K~( zVi3B|`=Jpe6{0dV6n)swwY3G#NP1vburwOaQ&Cej`1MFUq}k%Gsz`|0rEJiQC_o3| zj%lPre*_`1{te*Oeg@ftHcq-{7zgZd>~u-AreYPGr$^D4eH^I+5868vR>JKT`84}< z#+5r)(fH6~TT_AhylQ|6eC*HMv42BV_vg~8f8-U>chnw3cTcdA3}?IODR9ER#3CCc zX=BII($K-9gp=7mM#TQ;Lxkq5hJ7OTvl}6hul#@NI)jQd8)~h6nibwD0L|MhnvGT= zw4}Yfx_(3DR8l*EHg*;>rAW6kpPo~Ub=eynU31IYTK9D$rz@AJ4zH<+Dg*+Iu@d;1 z!c?lPZO&juRik0Vtz_qfM-dQ8I{-aHlrsy3(Z&_Q|Qwsx-;e*(>BUV`K4UnRK<8xjPj z7+?p{3VzTTGU=ru5B}KGqJT1XrVq3?o&qOnt0sStMEeVtAR#su!0-*Rqx(J^#vWY8 zj$|CZGUi!4d?crR3RNJvEYAuzn`2&g`0Pv{ON~X?ZyrK-s-UH+hcw;Y3T12DX9Ag2 zZB+*j%O8?{h>O~NIx5eDO)0B`_U`=Hg~A5ptC+Uel#75$Xl(-WBQn~8Lu;i!LBuCP zWm0oA0yJ(dx}%)MevOf$yS{!CR}>2Iq9a=zpV+7Wk{hm$;kJln=tcE@H$v{+$9;faUh)vhNK;)NO!aJYm~8H~J$M^bZq1_dSfO ztJ{k^K!K5NPvJtt@mXi_&F}CpvU}3yzfNpPSlVQ0Pu#vyS9Y>qbur2Wnpm<1B_9_r zuCAsiVG^4H`&WiSCVlS4huG?o)y85@MnJyCas$qU?`$ljs$ugG$!ZkI_~sOEpG^At zk+Z=AvsbqG`1ZV!-Sb$r74U!D8YXzVDLm7iDh;-LrDWlRY^zo$2hRn~j4P7QO?}m* zhaq>Gu$Q$37enBT6a~C*toLL%O#t6DOfi}{gr zWM@L_E_YX+zae{J{3AK;tp-nA2=7!>D&*_Azp#kEGr0eZH~egj=@$XBiSwHepz`X) zjP~7bYLGLUkfSEPnxS!{lhg z7?q~Uk!b;dRRjmQ4#QvQ16kjdzYplYWW_^JC1+!sv3k$QjVz5`j1AdSt~i8CzqSiGmZjL$h@~yd|ZB?!}GC z$?Ko^*vOO+_g?7SV;0E$50lNX2&^ESo&-8Ztu<9E#gr6G?0X;7B$$0rS3JhA9x~sw z%nPNLN@xMT=wt*;0OKk52rZ6xdISpFNsvjumEH1nskO@58~0x5+rh7kxC<1EE<$lR zkO#319iC%xnh$ai&F_DLK-#870Cm>6IJJlvubI0=L}k&e?%i8_=}f(GHScHP}jC_boh zk`Lp~HVSRTl}j{rv3w!f)d{ouZF(_@kIx&6d5DEym!(VgdT49VuzsEQR1_o=mie=L z@dpF^pU~0UoROS9&fc|+afxnmOlVHS>L5lbE;4{R!e2OXi;kyQNb;0G>(uApMvk zPAKgD*wY3J^%mE=9GGow5Xfey4P&IWVcs-Pww8p}!bqhAj+xkqhrLS5k*CY5s&|i6 zhTQ25Tu<6N=b~#8s6Mv?;WUn|6`=KDc{E*K(Rwffj`-FtNR#WD#f6|1E4?F_3c9p*9*h^8npQp> zRI*hLe6z(QB1SLTf;?2iMLq+f3>Exlt8r8xNdd?M$QegCL;)9XRx>|JWI*G6g6QWNCe zJyDT>1E$_Lr!64p(7#PI-8N4!ZeE_mffg)4(PpB0cs^9b~Gd0bS^cLA43bb%D%sY3;Doy`K9yU*U<0n%l`lul#_1j zLn|daevH-@4xXSTlP&+Rmi@o|9z96iJ7c0~>ZDV!V75RwhpVl!KgN_><#ZQlYk0;u znPE+LH|_SQHKVd%H}v%{`{?&$@K@vWkM9ZUwM>Q&V)oT?lG0|}zvfljFn;XhdUxZ+ zy^%+^S}f?x6^kP-u+n{&+=Fc}78s_WNLtVR7LoXK%fwo-3WeIZyqnb~FLX`C66pp4 zLoZ!%u)MGr;_8Bd)*f0)cF%LhF`*7$_iDXs?Dl+2=P*1ckuHfUpkx6!<|ev>;K6_43-DZOED$nHX%bNB=9QD z8nG-(XaO8q_uWf9vHa>Mw)Qto@`c_wnD(*Xdfp~Dcks_|2Ut9MU=GHhv~!p>0`Lv@ zi~)_W>Y!0wI?3vz&Md1C>X)1LGED}TXdm#Uxo1@*Lc^3 zhFfOV7FLp3(e`J)i5-z|@}_$tcu*TMgYf9umBp0lQKRnwH^DIlP*hL=HHp^&{0<%^ z1odcqfMyWw0H_NV&zVMiC-viYs$D@hiiS!I6T(3*60U>?xj;PT+yF|Uc*Qu6G4lg<(o|J6N|X>p41Aq71p{uryWI4|!$Zjz2|AOH z4p-J#XY!VeIK(>ZN!yv^O7A0zFf&4?OT0<|`QZ&DQZZpQduPYK4Jc}% zH?<=##yZ@B)=ql4eTS5dMXEx`9$0nci$S1-vJ z!9l=YGYeJyWiS4}Tux{XEbNnbYs|<=#CMDS|0(C~7Lp5SH=Q6IS2QCf05?2thN;Zy z8oE<@?2G#s^(vO?%e3pXLBBYkY z9%CJJ91$DcP#v5>)V;?2>ZSrCN1&#i*Ihcs|dcH$mdE}p z0{SeiWP@(fRaCIY`boegR>Xdf_hexaAHiwdc;EHTYnNc-vfewhw(AKmBt z|Dl%jAMnHeeyHu=kXhe&?)D&0G*yDvKJJVZ+busu$kO$Y38j$z@v7a|FLFYluQ?%x zoR?d=yVwl`qU6;BdG8;*-%annzP^6M`hkULh}iTcCC03!y%2-Hg+TwJRt0vncGS9L zByHx2$ty=;j{ZrAkF~GecY19>%vzc+*XCLue$@GbDN4fyuoCZa*rD@*=I=!szm{W4 z-Y;A9x0ChrE4pn(b|F0)P52L^rJ`wS?&FkSgIg4-B5GQ`WsDexIe}O;Pwch zLyU*U!uBS9duO5ZBU;g23+7bH!wS; zATtv2PeJ0}hxZ@h;Q#6lS*UTrlR>^GR`;pqy=iv|OMaWtaSJ9Ax3M_-Q`-AytQu2b z;Trz#{{GP?`1Pj$>W^0j%F|VBdrQ2u9NVQjSac^04GZp>-!Wp2FzpGwb>*?%^Gvpf z6>KB1#8PS^#X4^l_a-?E8df8kcYcU1bKHo}dAr9U#F6Q+3FmzH3R(_tQ%j8qEUfW~H zL!^Z(@>Z7RIA?3VbL;E|a^p^C+mDEB&8}om`vmxh{@DZnM2$NxIQ@@KR5uWhgnf^q0tjrDXovY|Q&-Bvh!)#V%S>U^hwC_2iSey(JgzR9-( zA9)!t^_4AZ^ze-~AG+n%!*_c$hy1>P)ot-90<#>Rua^^?tSB@5{Z2~npWV;TkK0NX zYXih@Br%&bLZFQ_jmiXBlO)c*J5#Ne6>4|Ey6VGE|*kX!{jKe@&4Y=@7?fO7Vl~X%U4fTPnWMIh8ZckQEV-O zutsCC?S-e~t4nVsi#>_`AzSM^-PIzq2f>a~BRIsAIk&&O5^~ZCh}ab$}o;wrj=cHAO?t|%!& za>p}F-9N8GSN?^-ylu+v74};yk_@q`kNZlC_%lA6!xMm>_YNDs-veU;GPuG*Nk-zz zLl1Y^%$wPzRcF@pM1nqw1ky2>3g^3?Tk6le{-@S=jBH7{GqB?fEV@(Iql1+N1dY~q zFS3REWcu+@6Vr_x(Wjrsn+;mJ`A=(W^C{oGaePbG^u5;h6kGiVrYVVrXy|lIV25O% z$oM_)s(l;lpO;^1(VQe8i;Vw=z4wl5a?RF;gQ$QsL5kE!6QnBATR^3W5J8$CAcE2a zMCk^Cpfu?!C?G)*kuDtp3B7lu*M!~^N(iL*Jsr^kF4o7!X->16gFP6wDIQcxab_pJzJ z*6*rJwf~t^*^hC9DvR6?kPI*p@J6*Yf77=M&h=%0fwQ3gq6Msyvdi=!kUabP(W`O) zKB~f>Brv~POTR4czxE^$Vp-HA060e$m91iiDd>ApEDnRSZByM7rb7SyzX5}?vr(pZHSFM|81c{^Z9p&m& zgW!^i^=Ft2oL8}4{)t9{UDFnCXMQ00yc-vw&uA8Zgd6e7Pk`bLh92}d`fuRI6n}W= z@*@5XLCu#Djf_#@z*08tIA>@3a;{rDtvEd_Y7~oL)@05$(qZI8yv$yu5r_oscKmre z{65+G>Cy40wRH+^%NUX-ho!}D)yrGVG}^G2aKOum!rrd(ZT=+vzQr63zRfW7MkrGGry_&Yx z4Q0Y5UvWP$JH!skr3F;rH~mnGbHlc_Q{}Axyv-TGP-`3pXsdHS5d2ToZ|X&-pLLd5rp&3c!E^R44rm$)_8$_1v#2F zGELW8!Zc*n%ipIJ8th+5p9!qesAC&_Tv&kyGiM8fzrXg$4AIT&DRC-y{ll( zKn5q#?i}Jx%-z@4ALp#gq8JO7SPpD#6`S39%x;hh;Px|tPenh^_x*lZ`SW*ESJwkr+QRek%;$29_|WGHGNE`E zo9EV&O%Q58-?lKqKUcp=UX3q8#>I|2@|w^W>)N;~I>bKorp9PFtPzC(3}Bnm&5TFL z+{Ck=Uo7x<93PTpxk1Jzu&JR&%2l0*L^96Zxrgf;#e@+7VjQY%1cgPkGI9X8*;Y6o zUjmfrR>i2DVsE=M%>jJW$FoJvqBLQ&!RX;o$qK>J(vr(DFE9UnMa2^~k@pct1RXOd zOte9u^PrFXzve!l7*BhXMGva>H$}>Zuar5(u>e2C6PDcLs z-SJ|$Lp*)qG!IqFesW>s*2#t|W)=<+;`Tv}QrL4Zk3>>zoo!rKy76U#`O~oSTH8 zykJ}ad|Ni#O9Wd0V|10Cj}j_)RQb@oBJ!jIx;AF!Hm z>rYLkZK($Xp!vUh7R*O_r>43Jp%lR6uKW=414c@joihx)KF7T)_uD#>z(%Y0~LktOq5~a&lTUXPI?&w0W<2 z5W83uA5wsQJ^{)Ni53RAKyR6gcqIu9-B86;0EXGh;cW!DH}oIM6DxbXFEB0eAD!1^ zP8Uuv&^ZG(cLZYcVj*fna^y-M+|K><%DC6dJ6tY$#=h@DFZ3I-jUJ=yI{{jT2@zoU ztR}K(e~rBSdc~ZpnyTgXilWEu7MBvMtsx+_sW_S*_PnxKFue={xG0%8PkYQ0dz zzxWQaHbop-xvTTc)Vm)Dq#XsAL=JjD55jzzviK(-8)bW-gs;F{SVSP8Jz$A<#j*`* zV0xYhEcwPllO8qS33Q~3cuTIQg`Ft6H4COxf;2bdV-%5H( z>rUdJ2L)*S?)uG;gPt3vV|itIw2wP|2z`@RiUk5po*WNSczXi}$1~;6uYr%jX#Bf} zvM?JN6^nM=pobQ*>t+N|=ci@8b-i?aZpeETE^rcCXC!FJ9X^`|E}FvW5qTodLYj}4 zeQJ%Zce`+S{kSr`UIuBO-<*!IE_4G72uA?f_1t6-6z|=;-iE|(-9=YI))3)tR6A#% zOob7ZU2u->czih)sC4!Qo6M*VnAEL+r}c39b}6ycYc~L0EPa0<2NhlVaZ2dSEjkxvA1`>v-*iC164kd*n@6w+Sam6S%7FJ}9v+4e*rOF*C^8hHwL27JY zldrV!$@GzOAjy(A)mWbL9kf*_CA^c)FrnL5s()l+-JdPi?jz6=RmqnaU?IK)W+^`X z)dfzOh($IaLlMjy!xL!6AWudne#z0E%&cbR7Rcy5qjo?Lfj>wk$7WO*cO_Zt6)=-( zn>zpWYqcSsLWB~NejMI-B(Tk9SJQKg$OwqW&V2)>E&rc;hN1!mA05Hn!UT-g_6Jhq z?R&Bwe%xfvUJLy8 z_x>eVSbMDm(FEY*+k{(#FojRmSXOz@4yZukI9EI4iWn_@ISFh5+L-kcYuY7y(GPr} zJ<-1-kNi_W?62;fKltQ7LGT9ZGO*YryhB$4x&9hK5_kTY`1BvSJVhxmOc?Kl)gy?J z*{Ghbmm$~n3d_v{vV1m$=EUXVK@6f1k9YoG&TX^ z2wuJG`3CMx=H@B@1RDygKTBFEH@>^%xWsXKwvP(=x9ih?$=V9S{!qOL1Jl)5p=>De zNb`gdQ*3lw*`R<9r>3Np;kBsaOnOzHgukdhIYWv3$tUo4Tebhlw)yQQ`rD8GdFs_1 z*+|zF5J3j+z>3{^{yPV!9Wo%-j zJIh*TUXfj^Qx+7{8E^mO)BOk5*8-r!zD0}!7JDOJ}rkDwCwnkck*|bO@9c~{cErd zaL@m}kt~3*=RwfXV0;$4jn%r&bs%I^aV`PjP)sh_^1&WrqA*7(mX zXQ~bWGBcbxI8$XqiXOKi5B-#p^Dmg%U9e$=JJ?VYZ(;YJkojE3hs&G-W-aUe9Oz#) ze7F%W{xE1ZTWsgb7ZKz5ctn*k<i7~-$Rd4^ zpY3#)#P;?>hDGICWA1WBfQ9=9PPkuftN(Ek?gZGaK7iUK0YwWpuFX8-6&duLIWcCl zg*7Q_;CuWWE-kG-8D)ODDd)&bzSqqDtf0NBe;$_n*B?FWg6#ZSXZi~a9;<;%>#v z=E!Bf$u_xnOWD`|7*FkAv&DY95<<7!_byy@ zKZmaPoO!*2$+3=LXi*)N%TL}URVU!v#YlV!49CuqJC0S5qe1}&^uHt>{LACx&%09X z0RYNG!rN1bm%f910Q74xGZPd*g+Y>avz{W?kJ$q5RV(T|s(;y2`k!L$wgLRZU|vyt zKjX9a7$hL`S+A`=q)BQo*0%LUNOfFlLr8BNJc{3u(**gG{_-M!6_$RxXE?}wi8?a$ z29EZQ^D#a5evr!f&IBuQF0|@4(i$0hJ4)LJ;*30i@Y4zan9Vl0lSL7@X@~D14Os|A5&k;h zUd0abYuI1@OzcKYFFDv5KKt-H==@=~`x5*yBi@`BIDp;_S^>!a(;1=(c)|}sI{NK9 zsBVF5`yYP`jQOnM1CW+KR(}VD&>-ms?tepT$4q?O#!eIjf8CmXU=LS4bj4z3;x)ywiq6ugw%0e3CL4vkV=X{?hd)V; zegUs3)3ESZm5Z~Fp`~jhC_#_c`56ZvlSo6mbJ=zKg~GBun42<+Sfim0);z>}l2y{1 zXE03_rJAt?2eb{~w|wzyVPiKfL9Q9>GF53R5Os1uVN@xkOv83G84urku7aki^3#{p zkt)j|%#((yy!BE_9NzBb8JrsrE(Y*>szbX)yac*A2%prPrka3T(?7nh5g}~44OGxm zPdHlCwm7y-Bu}<(lOBO17hkHbxk!|iO_)x6R-w0ieDy`O#lDy&dC>LD1wdgxMrR^q zVHy}l^om>BVNF%J;y#rds_&<_>*2%P4NnHg5-Hsp7=t0~*T*fKuq(+aM@(fj%f757 z)JYl{gb|t;(#-;>AB?aBZl&A) zqz^T)%URJ)bh5jqcOvEyzdyWGT4Bi*mgd)Ry5^nIH>96ll|>;|a@xfJr{JML)3=|I zqcUB41$6?H)<}lre+RAQz%Q6FM$o$j^x6zkjXe9!f%iY3c@r&3i2H$J#4C6>_V`kZ zQu!<4pe^zgPi6s~k3&~9E%6qv@hsiY(Hb@%!BWue0+bQxdRU%a~>^8noI0zl9 zJ>aUhTJah1TI638IhjyhMAVOapyZ>t94H@InS4{!{Js)t{naK;&jc?2@wr<8L58TEon)Rbs>L=#0^%Rd^;d{8+oIQm6 z?1~25Xx0jfy)Arn#jJ3}v0h84^S$iCM!ADK%1|z%(*RMh6U7)dez&Z$eax&WQ9I6q=-bswF7 zdV1Q`#w37l7<|g`@zsHyimb#|Clolm_89(SbjB?w4^sAxZWk`Rk>&%Xmnxwx0kJT% z{slkW&8BQrmi{C6PQm2Blq~lbYjy=OXJe!$U97uMfs#b-{!$(#o727)D>D7}>laEB zTyqS3&lf4ur+F_SQ1u}2Yjhe*=Bd&@z_d{f!MN-8w+Hu z{d&eo|FFnLUrQTZhuYGxu1L%|ceyj_{6Fd~v{z@}jR#B7EncVrns`#YAa@fAFqTyha zM;%6?H_Y+wa0|-ybo*GVi|Q8ZM3trIpxdoR8cv&<(1_jdAc0nwS?KwH%hTrt%pw*S z4hNbc?oVH=oI_oJ1Id|V6%z-2`EHMV^yoIRyVjb^`y!xc-Tj+$^0y)W$MiL18%T&8 zVhJK_Gl$Z^pndRfa-mw!hqJ;XOQ9wF#I`S5KwVenF<^gxTpqSkT$+2cG0 zW|UiIZ5}at;v>g49CN>Ac_(=d`R`eN6|Ye6z{6dk$;ugBz@0&IX}?<=z*}B_onKJU zInM?Wy%hsmdr||d{qU3(J4v4^&UXgS_`Z)-<2XRtZhAzF=h+NH{_VC=pyb91%C(f1dtQw&M{1QK=a{ec}Y{P~K zwD(@!LacB)4_j>*Q+OA?m8Gu0EUku9CGLwi$A4?SbO3g|=y)@Z% z_G{%r$CIU*t5oWcXY3G7sxNNvOCO6jY3eN8c9O`)fxDESlpJbY~f~%=Z*_tKRN7Hee{C4D(~&QjKWYi%Z=^aKwBw1OeX>%x?OD3VC!GP+K+Yi zPZ`uI2;&s-FG0PBtnUE2>I7g>TYKWXdq^PV-7_Jmx8lJ-0rnp7seSvCa|4jeahW4R zPSgSTIan z=WHU#LSa`w(=VvMmWKMsN%Mya9el9csN*gIF@x0Xpc6N6N%$<~*C_eR|FCMhTKd1E1BLnB& zXz$Xb> z_m-V}5+05QtC=_Hc__QaEb-8|&%T)X!hO=WU@|G6+W5dn_mTgH?lY|v{BxA)9OLP{7o*G=s6sE^B3BF- zx{SAYV!W_MOWE%?fl0CbURvMFje$o8k~_7^^?ZZqRh#nY65`Z$V-F4~UIdBliC2i$ z&%fk8zuKAmxZ2B|%pn=U+bt@C!Im0`wE(FT>>>)kP>h*7Tl5KJEDhduoZtcAeXtvQ zNZX{?(#070`q>v*A*6W%>DA8K_Dq%H1&(MX{~JSOs-AtN%$^GLsa$J8G1Z}tHuUL4 znWLYVbL+?s+3oFx?JjqOpEoJ(N%$#O3non>3kq*EzMRtCQOla}BJl?}{w8Ih*hUgr ze8@UK)IROJfiGpi*k$D3M1mNPld8pSu9M;^!G$dcNI78A= zIzyPs)30LvCRV#O@FX@X0&WNF7mJBiK#9Rq>@h0Z;{N84=Z&BY4?_3pb1~*YRy3r3 z*4_l%21)J;W!lc2xgNzJ?PbxInr22C>bFZeaecr6qI-8e2c#n~0X+(}0 zOd?(&E+-$f+1FftQJ`bInBnes&Prr&z_dIG@1@8FX1#o6i?HAzhQ*!jmmrBN% z<}K)UKJ#qm(09;+Az%{<-|Mm{!)Iai>Z5W|?hLvz@!4Y@?)H>B-#CQ$4VH$h&SAmy z3lie2Y~$lsMS+8dB{^x)q8TLuXMlvQ?w#N39k;RiV*0%C@K|Q}nOJG`W#!mQpmB=B zP}U_o6^-U^rVRtvcii6`<|j?RiM}@AJ{yfZ)eY00b7#dD?fZt@`ckc5-y~Gtly{th zsepyshw{r_VNB~He!Y8!-K4Yg?srg7uFyU=tIM~1Z1roeC5}_0@YX#f$qTwDio){c z_kLMBynTrIP4AiVAEFfU`Vchn0X1eADztW(-KBw5UX~%-aK?jGSNky_1iPj)N|K(| z=&0Nv2oV*&(T2LX%IV97`q>os3RO}KhCexEQs>?KaTEX((-UY&P%YO@i&b7#tanoS zgmsTZ{yg%F`O{)B>egP}d|J@M2O^Y#)9C5VY2|9&xip$MUGAdK_Fe*fPqB(JvOqw+Wv}eN;Aa^dvn<6N8)u zH_SY(OdaqJXu76aVioB08EL; zp%@?(Fl|thgZ$xSN@4tU3c%D4_XH4NSA_sc+Y6!y2(SWrto|L8&I=qYLG5G~5WGP( z9p1+Xgq%A2@1PTaVH0m3WSI(xH?z|4XFyPjbp-EH4gx`h1JVElB<)hhLka-u!qXv8 zKrs6E`Qs>KczjYt z?HIn|qt%RNMoEcgma})fOca()m?A;?!i<06m204Y{$GWHuLN@)AbkL))*YaXTJ;~{ zY@MH8a(~u)WcC9+sRA;DayS?wASS`jYEp(%uO z&PtN01e>O+={Fw~d(>`7y^X>ht)sx5ec^rlF?*N{k8;B_?*Q?_hfDlL1LEnQ%cAIx zh6+-LU`$xIw_m(?7yy;6u1=Qa2O7DQX*&CPn- z@=Qqm@vqbi3J1*wGG_;e)NaD#yRemx>G` zC1~qc4-$K&Hm>V35FOkW1QfQ0i<*pZyAD@zi~0epO+7089-F?gMCpB!!FSMsK0uu; zJ3taI1ItbWasb>tH$TW2WQGWt4g2-al){@70XhhP9>8%Z(hH!7Xe3j^8>QgifF$tc z40%HP*DnG7SG_`P2+nR>IK~~X@8jpj1sEAg3TZP!beX17F5Vf2LVQc{WrmJdI}Vra=5vpb2$X;3(6II+z1q zuehMbg<*^((^|Iy5)TY8k8CG*F79~B^V<{Tqbrbwv|Wn5{2zOQsvV9ol6uCjrihnt zor}J}t2aYAB|8~ksGRq4{6(sNwmCxe6>pFlm;{`OE?bd+46G#o;4Y9KZy@Qmy2e?F zQMRD8YmA-hQ-PbtFA`1D83&<3Kslk^03Scne5zp_IqLN-HlQaA*2y#JOK3jsA0KPx6z99X&3Qy`sBBS84b*4 zm}CjM#~&0@oZ<<{+M_oWkHl|tr+4=DQaxwvRN$Lw2g8#;UjIxx&OM}6XKxzm1) z1Zhkz_#{D&9^{G_Ln9mE;lmaC(^S~RH+d5)j{_$pO2^FQk7#XsL_M>^bAnocsIqYi zP{y<7htlAE&`Dv1dp8lCq0?fYVN2w7*L`~xPa{Xlk689t7(0rVKrhv-COlZ0`_@P- zJD+*egFS_(G|+yX*5m2rFA*vyKua}zzzD}|MQqt1Dwc|<3piewEsnX+xA0nJ2zO4! zUZfOW=mn*Hqxd=p4-Y(!cg_4*HrLDQRK=4TyPz#Gk5U&EZ;q)u9c`gWMPdz>JEr-G9 zsx9V|4eH6V8j1HVbykDs1XLZ504|E;3ouCx!`r(#_}*3vtyuT$E5aoNlR8~Vfipbq z;2GOsY4te)8Bi@N9*IG=ka@qUn4oNUmd8V1d}tY1W!HU}Q|_2!UOMsFj>{y*oFWKn zr_SpN*h?buH?gOhIESBjp`*bDm9b5(@0Xcmmb!;*`DG)$j>0>2If7%=mHVjP`R>d) zG{8gUA2^X!vAFmQt4R^xjRN7WW}3Qy=ZUnb#3s9tUci>!7v5yS7`gk{{iLshdC8E> zMAOW%sSlmvMhvOa*pV}bOp%~|-~d+(#P$*qVe7zdw@%(C$Lpzs-Ty0@KBAoGY&9zS zNig97XfK`dg&LO)V*E0$5{d8pa6F|BNq1kU&r`V%S%v&|*YuPw$8raot|GXBPsQzr zq0HY42*1#9{)wMte1UX47K>)w_(D)2UI=%$p?`b1#b@r5X0%2*aD2GBa%fjaf(X*T zZ-7!RJdQok6{;Y;08YBh=)ca*weziRt`ZKG5sN*!wtrUHk*Xv7qF`_E6^}yfg?b!_ zSqBuK&v~&>mfFql;G;uW$N7O~^xOBfID*g+5--^P*v@%iV*Y^7;#?ctX1{ea)z|Sq9E4g=<=A(W*D9oMaqL z9nAmhR{XlWp#U}rk(HgSKcfj8dLuv|pzmIGV#8ywx>+ah57YvzM)XK%Y=W?>$j(C8 zeG~P?8n#!uA1l+o-sl%@t}~OIEzpf-aqE#kStWZ z?=8otEJYnt@V2d+JH0BlS)6Jh%pUUW{w+mO<*A#w&IA2xWp8%ykkeRAm?fw4{)chwF-=8xafbEluoUa97bDSC@iDCS!N`)UR zCRJ^n5d1ad)6aIm&*Bz!(=#D6K?bUP5De&V(Q5*?n;eB`f#c=F{u7M(|HX4uUDM3? zdaOK3c;nfe)$PDue^S|g^M=yQ^$JW>Qh#P=@BI&|O%K;6&KyX-E}-D*wQZy713WQo zc;U-kH9(ym;t2l}(DnD9F&>6ELZdm!FCM=GV4Wh!bGJNME1L+tl<=I0*$m31wzJyII>oiLSFAo-LNm?>`6(AOP(FM{V|rR|td{lkfO zufjSu-x+U$A!6Qc{Eiol@te zA#gh-9qxzKm|*8@Dtz~#5#y8NQF><%P=oL8MGi~755lW{D;y3|F+-U}>_S7wshYtd zfs*)(Xen;r5`s6$ZF4t($4kLWs+9lrl|eqW8xL>3B=taNCex3MG!@eM0s(h~cxp=z zqxJb?rv@e`ZVHUlm}W zZ#sm*c8XuO&=Yix;m~W*;Gq^SN~ed9FEvbaVJ4AOL^BlWM5F$Yc&&gF+xbZg%MX$g z^UNj9uP8;X>2B}NP4-doVU^ZeIrle;#5p02dSG5Kjj!ps-6>(iA->{ZM%lrxY8zQ| z<>LWwFTTx!h@>+VqI>{N*EsS(E4m;X^IQ+Z7LvUT{piA06+K}(k78H#H zZIaItp@79)t~+L~9zw~FkDYU87^ThTy;ojZ(Z8`c`oQg;>1fwc2j{{wQOplPR0tw< zh-nzE7CZf>kf_>E1fR{EU6yn&wUWb1ZoDPpUtVoCWPc=9t~EWMx+z035ZG!V87&C7 zHVX%wZO<&jBl5%wP_vKu5;yTJq&$^?(CLa z{0=f1am~M55=nQoRA(VNTsVr@?CDUc zKJSQ8%r+O~yh57H1x$`y1E_8;9UY|h^slB2yve{|Xm9H?Zxd1!@pc_E1$j6OK@3x; z`Jw88T_Q(ywm|T5e?Gnb{!^6pqzs6Y%fk6(cu{E8S&uSaLQ^7!zi8ROjsqNi7zt`c z;tdHFe@p70MA^~D+fNrI^Ht@R2l*M`;i z?2cyS<%Y5*>01UGL^`|Gh9I@vUzg-PSTg14?R@m)obwDAeh z{#Bsf-U-2Burtlw8qv|yspEmsr^?FESLWOAosKTzraKzvdiFZ>&30i&rRgEJtinL1 zzkgQsW}uxV^A2~9-g(nhHEPw2;kD|qnVccVBr%|s!g!i!hzYH5LP4Gdvi8F?Ke!l@ z-s5Ow{q97aJnuH2vfgDY@*sbkr;%wxVqs&s>-@L7p(a9iUEl5nnieBh7P5DU7Z8hL z;aMSz+4546DOsmmXj~=TT_lYQ3I!2L&K>&og%yzUS~YmT42Fu#&WzGWmQ|;Zi>M;Vnl`cgdH218 zZ+tyw`+*i}IJFiqy~g!LgOyDJ{MA>RfV{wff~Lfa zg;>cVmp8iL8@@Zuwfq=({T4HRWtSkb_m#0RX}oy6ty}yEn%3c!VfiZ|ACaTSx$TdW zMX}!mL~}0q-y)0-<#S7w<+EgUdRLfL?2+9cEkG#(`vDDN#)#=tMcCFSWNJL6QNOGd zpeAGeAgpV9!4WZ^WN3EU)3%MHMf3&vb~VN}K($o9(t99$b*5nhyqdBuzfg3fwk#mM!?y zX#-hd755fq7;L(%JwfInhmEH3j+7!;9XTB$Ww?-KPRY%w^0*rZ_tWk(r~A&h1a-c; z{h*F1O>yuP5#Ei_&1j$BA4`bJb5FQm1O7E4= zj+Gs?d-jS!y;b!ggDU51n5?i<`*)CyKx;-+3tFJtfTnP*YZ43IP$f7Kxd)-qwX)=% zX)br#OKa{A4)yW6{0RmE4)Kzbu+ph#ll>dw9n~SyAr~4^tF;!7e8^vE(=KvoJR1J^ z1%^b8?U^-7g->(hD`xeuTF&oZChx^yQ|&dhN>dL)iRQ2UaG=+%V0j?28F zah@Mg68$Uvxttgiz7iD#?g^<#g#!b%s=x=lUzR6d#7<$tEtmaW=rjcjf<>0%j^9nt zeAvz@P;L8=$zDs@u%RPo(XI1KXJqSTMyY#AGSvbh0gU7mCUej9J$+kITaD<>1s)bn z?uDx_2n{W?wU&L?v_J>Oi)FRUoHvnc^R}I@jrR*O7D&PeZ!VGnZJE;6_5Pd{9JmY` ze7{BmB^H`CAjRVAFG3_dN9@HXP_{D2ZEA2eQx@kcNGyEzSr-{Gd%sSAtP$INYqXd$ zBckV~;5PFCZydZ>_dc+H&gH6>v z$yOm9Q1Ap)LG(iJG|HFZtA(?9WDfIslI&Nj;o2nY`v)D9+6EUFG8yhp<4EAWz8wfB z(HHMG_x7r)Wi#Tq+Kdm~kue|;K8^7$I(_eUWsEG3Rg6cP%XA%P5hDI^8Rd~$b96uI(O*I)YdAt(1B!g8@Z2r_&vV4%1<`6eEIVT{y zQSKoakU?OCR-okanjF*wDgm&}V0Z}c5cy^aCue(kA2!*yaueE$wRt8!yCsKoe?qt za>U((O<=r=y~Zlrml6eys`1LGcLm4>VWnIoqhaM&_ns{TQ^iw06K&QtjH6cQpdB#v+kt7jWq|m(~dBlKOsbB>Co<8$Iruo?&i{o*v0$OwTtAal_ ziZN*49|^do#`f~dU19iI+xYcN_Io{rb%^qjCM!zx$LAOyTq<4$#+u_B>btQ^(&~AL zD-?__K5r(;!FlamywL3rQ9;sLj~Xsr_Y&z?p&8JBq>+|OumNCZvvmOZcLQ)6`oZ1Y z?yIU=!`*kjp%ZlEbSyo@xH24K)ZBZD8v3LCQxo`8Sr+d3hdq&i1AV@_{+ z1&9tD+COFDY!??@XJOX4ZaycN>LVL9*D1`}7n-;3#>~QY2gIG=d=v&n$4-QxN16a} zSk9!z*iiz?q!0k(ip}$TzI-Z@>!A?vXOvc$yJWoXCwM@RD$L;KK35NA67%CZ@$ycg z+zl+_1}!0wc&fymop=i`fXa7oD1T>mJ<-eGS8~()qz- zCms%n9~RD5v_Pl<%6fn;_Ei0&tM@lMHDzmNY4RmcaiQXlsN2T*&IW&Ypy^?fDSB}} zKI;AKdan#A2kz2+3g^Y?5GoVPsE3Z_T3h8Z=tn(#XNb7d&Ud}bM?o|^L((MFj3fW^ zsG0S)TRXc!O4?UO(J-_;k(JV@gjb*pe_=M)5zMUp@fudJD=f`qi!z(?(>+sMx9d8_ zY2poVZt@>EZanIP+=EZ>)k@+4Zas<-_;`+W&K8CzTU4v15t__wkJ~fl60ubw)wU9^ z4nmkx?}j%JLdQk>)-2wf?h!xhk=r1pLcJScObmp>MXsDJnH0Ty7?|sN zryx!+@^VXCL07$naMX6|B4h7qTGq>>#tW7wJ}3&=jg?7gwQ3I*l(`x#NUX2lew-4K ziuzFs*nA-H@XWglB0G~d_ziHk2+c1qf)nr!gr*31jTlV*`Y zQAQ#!2;yzer3fbt z*FZ^#Ps&c60_;!TGmGkmfj**Bf(|N0A3nNO@>IYqkzu+0L+0J=C149dk>S>|ct9kz zMt5$8*?KKGeMCxvJmgLIu>Id(cbs0XtatCMX=<5j4-B>3`K3oe)jrLO ze>+D21TDiNYS|}!bI8yW7QSxEpJWo|*0CL~`R0|U*S-Lvnsw1WS5pH&Wx2Y!p})R5 zM3|4z1pOM40*DP}@OBvK^>@%Z41CB_=I`gWeZZp-Ft&f}G1c{`j>sSQVY}cg1t6%W zwu-1QTi>z8>gnO3chSmUZLWXY5*P$dC7Dq+SEcDiZR+@%iO}^DiYi$zj`=8VV1D

k?DG530+4eph&f>^(i$tl%zN8K|`SrE__61ld z0Nj(sPGhJ>A_JyM$|6cQ4*H&_f;lds5J< z>`*L2v}c7q4%A z2o*ShW)ti(d5g=PHLy?qEOli9gnmqar=qNDIna>YuIHK7mGtm7^mEAlfZKE+AMrfCId1~rcy(Z}eOC|4eA6-)nSdO@SkgW0!p0b)YOF^goeJiRzYOrGgIgQQZ7@Jv z#U5ZgTtq(yGj9@v5xCfr2&xy+DMQFqjNOe9(jBZ z-UVV;)LC{RV>7KlOFJ_U57=_Iu=Nw=(b2A6-R_P1O5$dBoGkQr@@|xbfM3%}URt=! z__gc-Xofkkds=i5=9fpjjOW1muhq%d3VFzt-9Egg6dgT3N4BJ%Kch9DDD*|Js73lA z#WMT^k!P-#%1J&B_=R_m=AwQdC1TIuX_Vx$9fftiEyzlu2g(xmt` zGXDFr`5)&0{O%VhmJp><%_3^FL{==k=?l-%L%9@zF|pa)X*Ge0OR99er`4MJsALIL z#BoXJ`)XhQ@J4E6v8f;qVorp|YE!qq;*4;(hna5o$6d(@F1st-8O=()dd; zGFZ>@^{u;iBfjNku{Mp!Q)6oup07=Pxfp5Zl!c;Db1*n9=<%)$#WQTl?DQ+Fd!sy?O=ZQ< z09EApqH)*5b5!TM&iiPMsYV#-f)-v?e^zLHn!P2~3UGBl!U|b2186!i95c=f{k46y zZ0hWAd2P1~droaOR*C(Y8YQ_AOuGkRhI+}q^f`B8rfWpHQ?1bV39)ixWBO9g(&M1) zs54Unu9wixL)kIej=7T0iB7gv;nSwWHL!! zH_{>!+#H*ubit+$kwJ{1#L)sGu~3XW2}t#J7rZR&Akk7u^R|iKK~=3w2PF{P2L1(R zsS!*8)1Td9wKlh2Q#(@@%5zauKA)#6;Z2CPSGh`M3|n|-;t_erEA|U@Qd9;Hb2542 z6{9U4yc5y$DJBKlU>K5c(wN!`fcQO@R97V&e!n|^rL^+v`J!WqVky}L{8P)<9=;Rx zI~w;uBfDAmYH6}eoyK6JmNdCw)6a%@ii)hmiv+-+XCvhJo2Fx7ShK8y)G=)gxNtf4 zHYo@2ZWpctKs=&@6S|jarJUVjDa=cmN4f`nUmva#cREoaWpibiH2<` zFY{Z?j&jGf7L#X6(ngQ7+%LAmZ;!k{nXQKv5Vf%~(t4R?oB3$Rxw6KdVedXcgCj*I z4AwpOkJOzsPPo{D@0~WgZml!l34ip=OQ^hX9*K&Cf0l1_z1-Zl1}W2S<{uw<4jj@d z76$Kp4addoyKd;8yiv}-$qTAx`S8H?nl|6DIZ=`EC>fobuENS_=QZJqJv_Wlt*d}? zSZsO)$N5ss_0yQ;)Rh;S4~lv@t0L~i+eg$1a3x&PzVS^u_`JHzpo^_c+Ct=IHGpds z{2mzYs$NFolvW^zbVJgkP`l+52X@^G?lGKA*YWihcHD|31 zznGf3r4V!aOjJ#RqQZ$BMZvU&_nA6I&r`>B-KHbhsTFI@bBp(S8<3In%t*Q#^T1>Y zfV#5dqcltfG*;CX_NAjB#{U?2AwW8pB8^W ztNh8>+V9>OiuN<6k9vxiHQPs?B{>P>g?*Nz1)eljxZfi$1|Tt1sHvHT!caLio*{IM zg!Q767rm}PO1E*^HBiuQM^n6*YsV&iu(t5Qjpjh%#-%q^p=`JMKIFr#M%r_U>jO?1 zI9~b==LNj5Xh)A5js!0l8M@fV|E2kx`Te_fkch(W?fZ5aZ%_*ymNSv=?WKs>sow0? zz8eiD`ZiN=e>0nicJjc?w%ZKwc7Uw~*^RLvhc76l%o)0yY4nq3K$z;1+X5C|GX zMX{+djX0Lfc0uI7oVqDmYVEbyR5m&ek^^ zP-G+#Q;v<0!o(rF!UZrSenvnE%&|+=z<3~73*hZVJ^vYPsWhuH~cEd}?`@l$%iaAYNAm8O_tg7Qj6CuA3kC30)gGv!c z$z9fL)x~ATHCX=6e49?9QMtSfd9KP^`4B~2Uu6IZ|`8_x=x0e5Zf=2-Ar^H%JW>BUAp=5o_%^943%W8NTb zZhWhc#7MFj^6s;zW;S?_-l-1>s{t$4wp}pr4_)k{5L9c6REQ04=_Q=StJ-WF&n2`# zX(yd@v!Jub$T^{o)IC!WMr^PS>15!U)~XNomlR)y%;xmi;b~Aw*12(RyQZ-qQ5+QewYVXI=o^+_&3rLj*7aSg&5oO1v^Gh!4Ovq7uUohchuVMlKU?l|-%;s|4P^ zs4b$Y=%pP(mqUB!+OClc@|Y2Aa_ZE}=;d~?<~^m}5OBqQolRu6Q|6k*7{(zyla{+G zuoPB#ZSqJFP!Q(YR?fPKv=Yi#ADB3;z8Z;_&Jamko5KP0*a)7}(&e$f8dPUu`PP!b z%oi;YDKkaFK>%O?+-iEM8X+J&JokYxz^LA>KH}Ia{d9U}Ren(LJ~o0f+a&r3cjzs@ zkOkO6z1hoVn`937%tFG=IzP@?#810; zOShhivv=a4yN4D&W=_TR@>89Fm{GIQ%M32vQ=3E2oo;5V^+r!~;WN>_pvh$<70kmG z1!3y;V3^WEQ;6o*60c>55X|2R>2P#BdDVle*Sa26XlQj>;{$1K!94<5LIpq;ov6(+ ztRAq-%jIMC-ac+Lm0lfT^SNkHUU5M^RF~;`&stnglbDzoQxx62_h6G@wk(ytV0VU* zw6}-qKF%5cHJ|7}N|A97KSaf-x;^HqluA3vBHLWu&h!!l`v2Pd?x?2THQgvE2x^pG zgeYCA^iHr*L_k2PLQoI{r1zF!0i;GyK#?K>B3(KN66w82CrIysKtc(D6z_J{+%xB% zIdkss&N;JY?plX`SPS;v+1dO3zPCKj^S;I;R-Dp_FG{REFLUxuJ*+tU)hfz>p$$J4 z!00JL5e^XJ$YsVHp(5lU;Gb$o12Og)E^%60x^>ZLWks=9PjyAhMUIuXF5f-kcsbaj zm)?%uTXQ$M_Wm0(hwQyzcply|TTaMB6o*_NX}lDWsDHC~?>)u4@8N+W1hZ(nQy=Kr zaRC`Nr!airZzjN3mXvs_r?X4YG}wGj;{vsJw#u-z^b~*>ur#=ktN;ys2LBi~i3q$*Q~A*WD|eZtrbweb;*OuNmn zRz+*gxJ|fSC%TSq)vK_HbAWY|M;o{dmYd{3wjdsjtDI&(GmoIxxfRFNZ1;5;x{h%> zJ-an;@^mLrR+9^;w-{_chv6#W>ga&i#?`xWu;U+H6i-Cl+d6nS1hreC5w3k283dxh z|4L*J1o#Z8)up*>Dv!phiZKdfT%m9D)%!+!3oqPh&lXI78}{MsN|=Tg(Fx&8vZ9Wh z!j~{AN47x169)1E9y;AEz@%o+>3V%`QPsRue}1_sA@MsXH0s|%`oAb@fbN{6LpHEx zk+pFK^>YxWrj>R0#o^WAk~g9A7e3!D&*+tX6?_it3_7$UFyPzk^N8L+Tv(yl$~iz5 zMA6F&4@Hol`+kyDoXowwZ}cpihyP0E0-zXK0|X7yS;8{`mU?sPOZM}sPH;A2c%F^3WPxGQS&wMQSk`+$Jzp$4{XrjIFOyoEKOdg{@&G3bmmtOAJ5?ASFI7z|9 zvckoG_LQ8b*Q&S#+A2VIf9Ihu@)wBb1K0q!9YKc9^YG5L%&`4cTzb0XY@KJ}DAZRf{0*-DoDGyv3fVi2U=D0ZBkT>}N&?yA# zN_T>5k3-iN*r;3}(jCwT(J;{5F-^w5W=nMav+Rh6rN@Y*)}0V-9No)(L`0)fLPPsl zgT{*dy8*Lg)<-`(@NI8DG z>nO!;Tay{lECn1T1au@o;vA`DKpwUu0Vo1u0GFkUTFWK_YGM>Sf*b?r0Qi74XY76? z@(XklvE4TN3&i(Zf<_N0@ZM9Sp-q`2Jg}#9RswQ^KX^$9BlTzXEZb(I$a-=n_x|=t z%#0{fC$^!HOZOgsm5g$f&@5NT;V%7LdQqLylqN(O=pg_rgT0`tZ$R1i;_T+Wk8EO} zY=7Wd`H-8~Johc>G;!NSWGaT6++D;0|5e5%uXOG8_ok!!U4K;%m3B$OtM| zfS<(m+d~#<=%2SYocQJN6|7Ibt@eWMY`8NhJvzmism0bfH4_iSQ1zkO+%t6w~9g zR|V5<^&V4pTO09i?v~0$$#&;oHR-tKNirjC4tzlTMUJ?VMK(Xmu4g5mI)ZirRA7I~ zb2tatg?Qp4`Q}K2#X_8HtICKX=F=(HnSRcPUo$w~Hve$E{%;}}zzV)9TN~vlZK_*eLi0iu;%wnlL={XcyMp)gP$L$*699z<* zEL~hIuA#dN;hX3Y+VJ@$CYrXZrr`@riIN9hWbwYHSgL?#f^ZM zY*yQ6-_%XP06n|kf2QlR_No1X{9A_mx>VEQ?0T&G-M~rhS!g=^y4ora1Cwps$Kfq9EjaGAI#XBk4MT1jVt}BN?a^Ux%2|0oM`6zr1418 zy0zwatnXZ6h(?{g>zhFU3JZ5BL{7>XhSI(E z7ApLt2IL8z{R=(B3(+0N$f0UoS@_HwAj+xW|9>+L#%Euod_0GZ;eUm~?>EV#_+q+UkVfVH; z#tvA;*6X>6?T6OEO%eJkM}0ZGk0W~>BW%E`S%X=wL zVQ>B_xc!o>U0~&ec1UNuZ*6^Fhe^rKD6!Y@)?)1a^yC#?*o_b)Kx0g)8G-YYVZ<(y zcF!k0*Bs(&ljH%tz{W$f^t=EH;bJxIUe{t)M|ND5ibwV+_t?S14-O%!V*n)V2;_SH z@nrQyL#bOsaAARzAnVN`aSvU&ol}bsJCLbHp5W5An%k)bF;nw=iw)X>1G@5qBuqoa zTcR*wXcXJ10v`f~QlcD7PrezI^s&BKTD2{tiJvNLc(N{}v7t@NR8NDx$zvck+$?3M z<@;#d?o&z~yO2*N?q25MMH?W(D{G$m1=1-M^_pEk8ICf7@iIWqg5ajq>dbj}I_ekF>(r(BsH!Ri5C$}>EDrVs*E#+B*)dSKaDq4dFpPX{x zbpZc7{Hbd}bNT5!!kg}LkauT-g9jAbNmW#85aMau7A7KK@AQL3{V;v+QHQsDQz+3u zOsg}Ki>QMdA$ne8zwX9+d;rr^fEwzf?PoA{HAu;Z&1D?neU6Lin2B}XOKgJc*2RFq z>ccdU;MM1~x*4b1&(g2AEI1n{8u?qtJpekuYQLg3!KLwv0QPN%V4WaFZv}sDP*=#$ zT`%v|W36N}>MnCrdle`4bgllbtw>4i6Bf@?CDoGUx18H&$xKHaA1SGM0fS=t3s$RM z(5Qyb?9jxg#Jeld_Cp3#yDhLSU575^Qi08ipr8xz>DgG#ADARLi&EoneDJeTg2>F1J^tH!ZU zmL(&_@R@`gWjMvAOm^b#0y|b253_C@@=?uyS~vc}FU3s5;{3HAIs$?MuRcDReO(XQ zHPd5RHPSk#XVVg=*CmjE2JQ{0oh|kG0Hvw*BE;IEi?}YUas#jq9!cr}A@qk7)>QksA^u)vifFcI!PvM+ z$;?}adq>|mU2YZHT#Y0mi`34StZNA4+`<>sPd;8Au2NS(Z;dW0n=rQ*)spE;INRFw zW!Z{02Vyk1F-JC4!0vI2_|!kA??TzlkMcQ4ZhOx8s`6-~&3ixIAn(bqjp*BE1A)Yf zkFI63##ZT-l|N?^f&sM3?LC5J8faH}%4IIN)-^<1F={J&NccVPeY4z4=Tz;_KNTAC z0vkm9iI4tU$rC{p39#$nic4wd=5d~F-RXQ(A8LWt%@bJcWlFqJ;Mr2hsx&wL#rHxu zD721rY8}93GxF!ugQOU;tq9P zqCw`+hrmR{5bcH<6ygkJ?t6yF^r8>QWESkTZQ6IH?umNC}>)&WJ2gzdD z7DWG+ApKA8J~`p(U0o#}o^N9~8(pBpa_r9loPU??0igjaeB!HQ8A+XNhbzTa%uQ=p zr7N63wjj=l7mu&h4(HgG-On^NDU&griJ$P9)`Xa@1nBpa|e8k&w7T3k)*?`Zirz9cy{KBm~}y-U5nQuoriZf#LalZClg z(TDh(ppZyck|2B>aq1~<_B#^U$TLFbKgMtP1s_-VZu29R<3-u^OSaihpT%_0e-k;5 zAl?PgBk<8>$O&Nh*)%|C;JX8a_YBXe3|9fe;GaWLNDY8t+yV^a4x&j1x|S0H7zoNg zlN<#k2clr&WjYEA^4TxY!PYKdRljw};lPlNfE?ljV0^C-KXiydG6CO)9x?&8_y6M^9#?FV zB8oKTXmkYQW9GGzXw%fkGM&_h+f|cvU9Y{~=ikue9VaPLd#K!$%wk{thpxaL(*~Ev zU$T&&WpdV*oxZXp;H)d%k>R)X&_JexfrSSB`5(!D{#Uf~ul!O%#iCdn6mfdEs6C{c}Pwt|MX} zah|AZJsgp_DCVke>=L7s9eyuo!mih&z{;u@;&=L$4pKmmZ?!QdE=Z%@pzleb(%S*X zsH2g43kTFjz)vTvaUlQKiM%g~;^a?iqk=1!an^QUxl6_paNb2mN2TBY1H=G!+36c9R$t&Sop zg~_j=Q3!(Y2K4Brgy$etJk)4?0reEBeRzRx6IlL2nRD)>D4^ABH>R>3!3C4*Y>@!k zT_Xo}9aE`O8nk$D3z&5cT5Z@oS`~R8*YIl6=Sqv z>BF0aPXtDFzA{f@DR5zp2L^ZT%X|)i8U`$qEWK|CRCzC~L73H8KN1g`4|v_)l~{E6 zpIp%jH~^n3{5gK}5xPavtF1~CY#Yz31$y$~9a^^IyxW5W#(z=8GR6#3o zAw|WC&6C4aK;_V2i(iUVV___exRUdH$T(hxfypmb&Ei_1tC=yw-V7BsPChdLY>t2u zqCZ3Fhl$k}V%7Y#921kCweyj>%F1O0F;?mRyI;7&X5PPg^qz4M@UGh8A{blq(Q?*;tVVe+;R@#`{sNgakmT-6T^A>?t$k3f*9pIIs6QdCTXwv2 z3&ZTPp+}EiDaTj+$Sf>pzd= zvVKhPdeqqE=^#n27B@IgZdC<|f?WHW1mgopy;3~mdHUMtbq%bCmr(^3<3l}N5OuT2 z{BE;cRmm|F233+*`+bCsc;o z&Gt|`MPIC&Z=c)vn{?!Gtqro`J4t;LdmwOYH{Z61boC;0>|D_JYC=9kk{v_&a*UIr z%OuR~QDHR!6RH}UO(!B&|G2sy>!o}$BM4Iqpal{(xwlW=IOs*}02oN`V!W_4&@KZ; z&fsgxzK-Bbo-+RV0V~|apV>+p&rR{rU{=l9m@E7F!=R5OR8<~opV4=rw&YTAN}oyV zgH1H5JtI?q#Kq+nb2eke0ij~jTz21;w`F{$aX6w@jIm{3@MRjM*e22WXJK*DOle^w zTFeFLESHaDi#5#hn%qU3&sF{suXm{|y<}S{? z195WC6?Pjx-i{4kxP25iWO!n%sDiHYrv4|nUB|a`83Fw3_p31Ng$B~gGHGwBP1W)i zN;hfdXPE+4Ru>RWs3@-47BI7oYLB;2GROBF<(_i-rFfl~v@Hgb;eodDOon{BS`xdA zqoo3lI;sadOvb%v#0yoU|C$1Ck z=q6zkiAMW1VIe2xp%EJAha1k!#qmnWY7H&y6k`{=FuD#7Y84esy{(D50^_GwMH9V} zSI*R_Ry?C*0b?1ZB4#U-pSynodYs4T>lys?!CH0}{G4)YUp{zu7j+K*wPUIx;gG7pd{r zhaK(65z1!Y5TKJCgd0Rpk}`Lnolb4Tc!N=XR{X*J5zkc_Hb7e$&wuSZbC<)#NAaKw z`*@Sm>>qxz>Dn^mSPPO8z~46rpwY)El7(QnIE16idCaUcwV&ibd8N+|tXw4LZbiK( zskTFz@eB;J>s8g9MtNVi4n3#FoKB=T>UTcyFwdbqh=C^qX*wt#hFj+t)zA8j=tBT)Nf>G-wo5;1mM31_Jiu$6s zq3YHJUOrv)m}IWu<1JmO%`rhr{yZgjAWnK|UHdk*joQCKN=72kYO{n2q!d1T15!s& zIE;kWbq2e7b7q_%-Gukr@DT?V^!eLo7>L6Q;K-vbq{9=t_XcN;&3nVC?vmy|FO)$7 zV*N@AIKp+6x)pDaF&mjWuO{12n^D{io^Gk^ZOAEbo62~k-c#?qtFC;M0_n+jt74gc zx*AF4?IsbG>p4AyGRt=1p_y|;<+<(1OooAGLmEpSSzR_+v_Kqbl5^PL9A(F_4|ZBH?@*vQ)rvzF;(UHR%H0l;M=O0C;(8*QDnc z$S7{Y{o8Q7a|HiH`>pZXd5s(6UYUSuM`SIt^5I(0Xibb%gYd$@=}ws5*po#Wj`sb~ zqNfSvoY%#~3dQVWS6Ohxlxvvr+m}Z?)V5{`?D(3vn^k$(Q>0q~*&V6s(PhuI%xBQk zxl=C^O?S;Z%}O<=Yi!dt_B&gEm6C!rCf-9jP%^0}z)uk;iyN5`%|?`(CHY03>zgo# z>DA6YYGH4P>ooFPzp9yf&;e*EjtxbE4h{16@^@YV&UIgFJnOCzv+OV*V7Fjq*}W0w=pj3uOrr zUgX}0@yu#W@qQ5O_9zvz-u+H;X09vDyu^?GJ8xC`efvU_r9{l-d){&H_NqMJhW{Br z^?Pnpl|MOUU=m5~XabsxOqt~+Z%ZW%M^0y&hTJ78<7OAlB4I6%^JFQPLdzZ7gosJ( zeIQzrDdAFBADmmHJ>M2gXLOiF(m^z-YVe1MlFXttIJiqWi8gATcx_%^F@ay8PYEKF zou|kQ>5QC@Pg8$)F9aNlaflG*LuH@vowSbZfC0DXAw?ccjt>Y<(ilPXJ9RcQxnm^u zzlK!(0Wj?(ELu%AD`21{H4|VDd}{mbq|#PYZB|_#TlA$f*TT@V38T!TEq4rG3IOhL zjs!HubyG8H{W05XkL-(9p zw07k9(kB>8V_6ZuZ7_n(t*Ebgt?CeQQ)JC^BA*}s0jy3`C&`mJiA<7$M9;Yi@lv-P zAykG|ZN0rP#XR|%g-Kp_EgdK=<1x+fzdSOv4I9b_4b}k1g?1oYx5TTN&IpYa_Y&|* zts=`|5vK%6`En@fII#6wC7Bxnqef{TFoGYIm!onS3Su)#kI0CUh*I+9ffq2KW{XP# zc-w_I1Yvdx=&D(X`htaz5{jvgnJb&KEAEFitA1PFsWxMN%PW5SQWM52E6Zlv_964a zNC)CLDKJ18&O6;v8gP9+IWAZ->+@s}ZMEmm8T74qe1B}nU>lZiD!pkTh4F=7Nr~a3 zh!C_gJTeojQ5J`+7?U={!*CsANAc--n4{4C*M;U&GZQn-gad`~^Wq7H%=u>8TC_6d ziS+!mOlLfWhQF$7y-CVw3#Ds8(}nxIm4h&@1;LS@EVp*VizE_f=xe0?rno?b$qsf; zy9D;-ZW$P0qorfn#qTB984XubV1{)=zAoIsQ`{3aGb`u?!Nki8zM^Yu-y=+ouhQB+ zyPBntK8Q9p4d$e>&+pYD1O|(EZc3yMy za<}pOhV-s*B@@$ULG zR(bK+_XKoJyilcI%En>Z>+kaz_Y*tAJF2MTfC|7L*WFgR9SH=Ld-OTc91bqrH{z|5 zUY_Sn)l}^LE*mn$_5_e=>niu>U{t$)&ZKks+{@32&-gG1q$*T~7$7&?{`dsV{ z*JSVZJWxk|5L-sd0rz$K+e76hW-Bv$gT$REg{iO0e)g$=pPEcw0R2@`JiBvOSlY zZS6Buf{>qAGo6oYF7$a7Y4Db?b3gYj7|@v%njU>)X6{1B3RJ~zWq0*?1av1oumZy- zn^kA^Tu_dAx)x3=eOTVEP4O4TnduYtr&nnS6tVrrvt6P~$`7+%5|@7F)KoZ$!MX%P zN4FhDkx>KQ^bSfj#+l7YDe0x9Gt_OF{ZOODe@9jSf2F_wU(`Ey=n{#q33rKS7BZu_ z0Qkt{yF$!@zQVfH&9+J;tv_@2NnPP6dR+^1nq*F;V>M_0FY^I_*YO|kj5MuK7pch_ zLrONBH#az6(`J7AxjwN$lSDxRwHYtF?SrLHUHycR|pHv)2j6lpHAyQ41Jg zKiLm~ju)jT5Lv_+-C85=pPS799Z$(VB;U2+_;+N&;1{|(bwuxXMqMGg+)~=r`=WLm^WE-Z*xYGVU|MGm{47}j z=l^{>(o(y{kj8>Tu1|>PlXg?`ie2yP?v`yGY@*@S?;zj@`!}ESU%rF0G>h!tQ4t89 zN)8{XmuS#8HV3mW4H2dhm~LHWr+^$7k&(srM+G07DO$Q%bok@;$p^4AhRnQ{FY#JH zmCG+g3~@H^1!N(!O=~a_QCX7g+hg~*!kg3to~%B(M4lKBsyW)*?&~fE-~`%5YEN!Y zSD%<4icB*@p0ol~Jcd5`0>NufZ>O`(((4c3|$tP#h~ zsl6&&geJL%f1me|_G})GWu8qEM0{=cEgWl=+tBYito{X(30lAMhg|C4d_jMc-&QL& zq4Mm{$Z&u8Hvi-G^`|4j?|`;+hLhgr#Mi!gAy0^c`CxE}EiQm*-M_57JvB!x4Sk<^ zC+%62MdN*G?L%%+ptwF6Ln!#6p=vo3px27FBANj4KBLv|%AakAs7|Cy4+E8E3XkRye0iWD|^S?@al+&Bba#pgP&y;={t!QzHVQ z4OhA}ymawGN)u#?fndm1`W=Z5ZfTkS1!^T3e`y`(&{wtUYVLBEycOYfmfcfJji1R+ zgl@#=AM^J9&Z`{bX>0qI+fZX>(sBA+HqeUZ%H10@f^RcGG>Q745ac2ft-TeL=5`2F)TrKJnzEx~Ow@G{?>)Bh=I^}uKb?Nk|0P*Hn^*%hoC$BZ zZl98BQ|2Tehpnef+9olmWEPYGb#KQO2p0xXrWl_IiR+SA z``C)X%kI7|HGb)Se)|W@iyrar=3E59mWnsk8eagnvY2Bl+pnP}QgzUFHRwQX+k|A3 zPWtzaUJH#8N%FTu&#(DwiHdd}Rac7qO^5KOo%?UyJXnP|g`=5w#GaZr1>mvmGdB@T zClyi>7F2sU=}9>K*sM8wui`LHb5Bi(_%sGp<+yo+Y8K@x4~|37xoUS6Se23uee^uh zMv3+}IG1aLSVC~I5};5{TrlAo@g{yzhpV&&$~o!mRu)-X=CD)h>Wwz%KHGq=KcRbp zc`Qtw>F@JOf9t#b!T;e);~`aQEE~k zgv&s+r_B^$l10uTiTE*7Ij<9diDlU{FtWfL0Iz;NN(V24ADsn``h5=K$F>xe$peAE z8)^`_6ImW<`wJ9t2WSnkc8~(_u#}+00*$=uhu0^se?hc}z|pgN!9Zo{MH&Pc0u6jb z)@IfObp8$0)%{2vsQxq?fT^O${6Xp+!ktYBt{YNaL*d;4O8mk=L6TK_WDOw_EaC?y zJ_1i!QIUZN+-m@Lh|b3NK;NjpQKcixL}H77fwUV?MBU%t!f0Ow(#_k=^ZcJ*;-7Eg zpXT&`tcfVw-^r$ntIk_BZIU;}(=s7~%RPcG+LZ3|SwZ8-*;JQPH$z_m%P>g02{{on zH`^=`8WI>H?|y+edubp_&BV{$EzxlMM&43veBZM$sZldF=?M5aTF`jErS+Ou)>wgo=U(Iga)o=VazCJJ?>~py}F2Ot};4 zC>d4hOb3KI;)yIL>iKcW6SX5D>I&-;X>o13R#89F?@I4XWZ4$oi$4PjUCwn*h1cZ* zGID*iwK;|jKW#YxnNGgUBBMkQy-I$3eERQAKSKKO*#ugmDqhi?R~dI>eD-vMC*!-h z@rEn#fp;jBIKw^jr9_LzByuUro{m>HKjm1R~`XgcCIwFwb>wm5#|5o z1NcLcFXI{GAytYDfwA5xVr}Kw%|u1stT(R^zyR(49!Gl`>)#6J#Z_k<)kIZcRdvdc z+dpTg1snAC{V#(1!!&&G9b^{fxnyKvCO!E|)i_)<;NvypGm6cAQaq4kAUtv zmxPVlKo4rd5$<5Lr8GE7{*2-r-E+yn#3{qKGaCozdHvq8lqPj>o@C=(=J3?y9meq! zG>H(B1{e#WBR-xb#%s1Owg%P56yI`qvdwo56ys-b!}-HO*Giar!GF(#`Ge7k{pUwf zi%fElMn3zrFLz>OXTmgSagJCBU}?^jVkr$e_(+CEiZ#wLY7{Zgjpw0!`&NFfsBh`^ zHWy#G-P=cJnw3mxh4et?ICh}Y5f7_c#aaaBg|Om*i2?A_0fIl|^+>_y7FC)ho)O6k zm-1|{g=tN^syo(be5{8$UBeZ*IZv_z7#=0!ZgQayrzMixSFgUm6EC*Hcww*`f1|@Q zwoQXJx`;_r5&YJkW&q)gnnJX|s$r38>Nv)E$B5iP06UKf-1ddB+gMIC(Oh)>Tq59m zugAXrid~I!mXszW56Vd9(%=b^$7QzQL)^6^d{lY(u>!UOga9T9A~yY9$2XAA-ajBb zx&Ud-3LqT_gN9&rLUhsYt?oK9ctsw|Lt^$shculnk$kskD`*Z@8 zPQxO1~G4E2gv&600QC! zP)UpX1^ST|-~yb)4Wa4+Ohq^brRjP>Srzz!0B z6KccD0O}(*5UnZU#;I%yQ1WYlBq;$<>SrmO9T6y88Sww5RDetQ1knW4?AH3#fTnTb zaeyNb_^qn?=S}|eN&eF&|G9Af!CTO&-#Z!5&lDh8^UjuUhwW|dkzi35-$M?zFk$Cs zsSHsIz9az~UZV6oXm)33_x|mk{g!?xz8eIvIv*jXT-skiTeqhyt$Ae+u;k0x;=+(= zi`Sc*DC45XG#_vWWM&rvBF4nIHmv}$ z`B`A%);eMW1txQTBzko`8+7uMo)SwS>wh`Z@pEaA?)ra{R`3tV?^*t!H-w}ia3(P3 zbr@Gj;GE&4$vi5;wc$FRnRLUt%cnSU*w`F-q_GvvBQ5v7_C~Y<|Es$+Kt}w`ifn+} zdN+p+COjSwQKX9q37(^9ZLLrzQ|!@+@BF&NR$q&YvkCDW7r)_|2J#3JikZ)hLf}jx zZ0iuNOEn|RH_9dE`v5l3lUHg#tgOC$rJq{11T-<`cg< zIiN_igbjf_Ny6Z?K5H42(8RY|3p)4!M*Iz=@DJ6){$N!0_ZQy&|NWri{I<`ApQf8~ zX0AJh3NQ&)wGMJ-M;PbuS3Y?2HKIwZ&g-O%Y%1umh46q9T?9WCu-yh_CByJ%5id0q z@T%3=ha0!qT-_3-1FQ4i3)yJ5Tk$hwY0~&ij{BPaiU< z=Od+&N9zVRYmSkcfLn~Tj?DE>blq3Qk9iZaDUoCdNz293z4eT6TWaJ}a%gsuuFJw1 z6^Z3!g=xq;4xfi>-sLwFECQCpK%hVqM3Yh@e~8YgT<&jioHJll8!c$LMthYT9$k=B z&F^;NiM3k%5w_u-0(5k#Ad>gh!Ab<, + #[clap(long, env)] + shard_directory: Option, + #[clap(default_value = "128", long, env)] + max_concurrent_requests: usize, + #[clap(default_value = "1000", long, env)] + max_input_length: usize, + #[clap(default_value = "32", long, env)] + max_batch_size: usize, + #[clap(default_value = "5", long, env)] + max_waiting_time: u64, + #[clap(default_value = "3000", long, short, env)] + port: u16, + #[clap(default_value = "/tmp/text-generation-server", long, env)] + shard_uds_path: String, + #[clap(default_value = "localhost", long, env)] + master_addr: String, + #[clap(default_value = "29500", long, env)] + master_port: usize, +} + +fn main() -> ExitCode { + tracing_subscriber::fmt::init(); + + // Pattern match configuration + let Args { + model_name, + num_shard, + shard_directory, + max_concurrent_requests, + max_input_length, + max_batch_size, + max_waiting_time, + port, + shard_uds_path, + master_addr, + master_port, + } = Args::parse(); + + // By default we only have one master shard + let num_shard = num_shard.unwrap_or(1); + + // Signal handler + let running = Arc::new(AtomicBool::new(true)); + let r = running.clone(); + ctrlc::set_handler(move || { + r.store(false, Ordering::SeqCst); + }) + .expect("Error setting Ctrl-C handler"); + + // Shared shutdown bool + let shutdown = Arc::new(Mutex::new(false)); + // Shared shutdown channel + // When shutting down, the main thread will wait for all senders to be dropped + let (shutdown_sender, shutdown_receiver) = mpsc::channel(); + + // Shared channel to track shard status + let (status_sender, status_receiver) = mpsc::channel(); + + // Start shard processes + for rank in 0..num_shard { + let model_name = model_name.clone(); + let uds_path = shard_uds_path.clone(); + let shard_directory = shard_directory.clone(); + let master_addr = master_addr.clone(); + let status_sender = status_sender.clone(); + let shutdown = shutdown.clone(); + let shutdown_sender = shutdown_sender.clone(); + thread::spawn(move || { + shard_manager( + model_name, + uds_path, + shard_directory, + rank, + num_shard, + master_addr, + master_port, + status_sender, + shutdown, + shutdown_sender, + ) + }); + } + drop(shutdown_sender); + + // Wait for shard to start + let mut shard_ready = 0; + while running.load(Ordering::SeqCst) { + match status_receiver.try_recv() { + Ok(ShardStatus::Ready) => { + shard_ready += 1; + if shard_ready == num_shard { + break; + } + } + Err(TryRecvError::Empty) => { + sleep(Duration::from_millis(100)); + } + Ok(ShardStatus::Failed((rank, err))) => { + tracing::error!("Shard {} failed to start:\n{}", rank, err); + shutdown_shards(shutdown, &shutdown_receiver); + return ExitCode::FAILURE; + } + Err(TryRecvError::Disconnected) => { + tracing::error!("Shard status channel disconnected"); + shutdown_shards(shutdown, &shutdown_receiver); + return ExitCode::FAILURE; + } + } + } + + // We might have received a termination signal + if !running.load(Ordering::SeqCst) { + shutdown_shards(shutdown, &shutdown_receiver); + return ExitCode::SUCCESS; + } + + // All shard started + // Start webserver + tracing::info!("Starting Webserver"); + let mut webserver = match Popen::create( + &[ + "text-generation-router", + "--max-concurrent-requests", + &max_concurrent_requests.to_string(), + "--max-input-length", + &max_input_length.to_string(), + "--max-batch-size", + &max_batch_size.to_string(), + "--max-waiting-time", + &max_waiting_time.to_string(), + "--port", + &port.to_string(), + "--master-shard-uds-path", + &format!("{}-0", shard_uds_path), + "--tokenizer-name", + &model_name, + ], + PopenConfig { + stdout: Redirection::Pipe, + stderr: Redirection::Pipe, + // Needed for the shutdown procedure + setpgid: true, + ..Default::default() + }, + ) { + Ok(p) => p, + Err(err) => { + tracing::error!("Failed to start webserver: {}", err); + if let PopenError::IoError(err) = err { + if err.kind() == io::ErrorKind::NotFound { + tracing::error!("text-generation-router not found in PATH"); + tracing::error!("Please install it with `make install-router`") + } + } + + shutdown_shards(shutdown, &shutdown_receiver); + return ExitCode::FAILURE; + } + }; + + // Redirect STDOUT and STDERR to the console + let webserver_stdout = webserver.stdout.take().unwrap(); + let webserver_stderr = webserver.stderr.take().unwrap(); + + thread::spawn(move || { + let stdout = BufReader::new(webserver_stdout); + let stderr = BufReader::new(webserver_stderr); + for line in stdout.lines() { + println!("{}", line.unwrap()); + } + for line in stderr.lines() { + println!("{}", line.unwrap()); + } + }); + + // Default exit code + let mut exit_code = ExitCode::SUCCESS; + + while running.load(Ordering::SeqCst) { + if let Ok(ShardStatus::Failed((rank, err))) = status_receiver.try_recv() { + tracing::error!("Shard {} failed:\n{}", rank, err); + exit_code = ExitCode::FAILURE; + break; + }; + + match webserver.poll() { + Some(_) => { + tracing::error!("Webserver Crashed"); + shutdown_shards(shutdown, &shutdown_receiver); + return ExitCode::FAILURE; + } + None => { + sleep(Duration::from_millis(100)); + } + }; + } + + // Graceful termination + webserver.terminate().unwrap(); + tracing::info!("Waiting for webserver to gracefully shutdown"); + webserver.wait_timeout(Duration::from_secs(90)).unwrap(); + tracing::info!("Webserver terminated"); + shutdown_shards(shutdown, &shutdown_receiver); + + exit_code +} + +#[derive(Debug)] +enum ShardStatus { + Ready, + Failed((usize, String)), +} + +#[allow(clippy::too_many_arguments)] +fn shard_manager( + model_name: String, + uds_path: String, + shard_directory: Option, + rank: usize, + world_size: usize, + master_addr: String, + master_port: usize, + status_sender: mpsc::Sender, + shutdown: Arc>, + _shutdown_sender: mpsc::Sender<()>, +) { + // Get UDS path + let uds_string = format!("{}-{}", uds_path, rank); + let uds = Path::new(&uds_string); + // Clean previous runs + fs::remove_file(uds).unwrap_or_default(); + + // Process args + let mut shard_argv = vec![ + "bloom-inference-server".to_string(), + "serve".to_string(), + model_name, + "--uds-path".to_string(), + uds_path, + ]; + + if world_size > 1 { + shard_argv.push("--sharded".to_string()); + } + + if let Some(shard_directory) = shard_directory { + shard_argv.push("--shard-directory".to_string()); + shard_argv.push(shard_directory); + } + + // Start process + tracing::info!("Starting shard {}", rank); + let mut p = match Popen::create( + &shard_argv, + PopenConfig { + stdout: Redirection::Pipe, + stderr: Redirection::Pipe, + // Needed for the shutdown procedure + setpgid: true, + // NCCL env vars + env: Some(vec![ + ("RANK".parse().unwrap(), rank.to_string().parse().unwrap()), + ( + "WORLD_SIZE".parse().unwrap(), + world_size.to_string().parse().unwrap(), + ), + ("MASTER_ADDR".parse().unwrap(), master_addr.parse().unwrap()), + ( + "MASTER_PORT".parse().unwrap(), + master_port.to_string().parse().unwrap(), + ), + ]), + ..Default::default() + }, + ) { + Ok(p) => p, + Err(err) => { + if let PopenError::IoError(ref err) = err { + if err.kind() == io::ErrorKind::NotFound { + tracing::error!("bloom-inference-server not found in PATH"); + tracing::error!("Please install it with `make install-server`") + } + } + status_sender + .send(ShardStatus::Failed((rank, err.to_string()))) + .unwrap(); + return; + } + }; + + let mut ready = false; + let start_time = Instant::now(); + loop { + // Process exited + if p.poll().is_some() { + let mut err = String::new(); + p.stderr.take().unwrap().read_to_string(&mut err).unwrap(); + status_sender + .send(ShardStatus::Failed((rank, err))) + .unwrap(); + return; + } + + // We received a shutdown signal + if *shutdown.lock().unwrap() { + p.terminate().unwrap(); + let _ = p.wait_timeout(Duration::from_secs(90)); + tracing::info!("Shard {} terminated", rank); + return; + } + + // Shard is ready + if uds.exists() && !ready { + tracing::info!("Shard {} ready in {:?}", rank, start_time.elapsed()); + status_sender.send(ShardStatus::Ready).unwrap(); + ready = true; + } else if !ready { + tracing::info!("Waiting for shard {} to be ready...", rank); + } + sleep(Duration::from_secs(5)); + } +} + +fn shutdown_shards(shutdown: Arc>, shutdown_receiver: &mpsc::Receiver<()>) { + tracing::info!("Shutting down shards"); + // Update shutdown value to true + // This will be picked up by the shard manager + { + let mut shutdown = shutdown.lock().unwrap(); + *shutdown = true; + } + + // Wait for shards to shutdown + // This will block till all shutdown_sender are dropped + let _ = shutdown_receiver.recv(); +} diff --git a/proto/generate.proto b/proto/generate.proto index 8c5221b48..45afca82a 100644 --- a/proto/generate.proto +++ b/proto/generate.proto @@ -11,10 +11,6 @@ service TextGenerationService { rpc Generate (GenerateRequest) returns (GenerateResponse); /// Generate tokens for a list of cached batches rpc GenerateWithCache (GenerateWithCacheRequest) returns (GenerateWithCacheResponse); - /// Generate tokens until the text of at least one request of the batch is generated - rpc GenerateUntilFinished (GenerateUntilFinishedRequest) returns (GenerateUntilFinishedResponse); - /// Generate tokens until the text of at least one request of the cached batches i finished - rpc GenerateUntilFinishedWithCache (GenerateUntilFinishedWithCacheRequest) returns (GenerateUntilFinishedWithCacheResponse); } /// Empty request @@ -92,27 +88,3 @@ message GenerateWithCacheResponse { /// Next batch (cached) optional Batch batch = 2; } - -message GenerateUntilFinishedRequest { - /// Batch - Batch batch = 1; -} - -message GenerateUntilFinishedResponse { - /// Finished requests - repeated GeneratedText generated_texts = 1; - /// Next batch (cached) - optional Batch batch = 2; -} - -message GenerateUntilFinishedWithCacheRequest { - /// Cached batches - repeated Batch batches = 1; -} - -message GenerateUntilFinishedWithCacheResponse { - /// Finished requests - repeated GeneratedText generated_texts = 1; - /// Next batch (cached) - optional Batch batch = 2; -} diff --git a/router/.gitignore b/router/.gitignore deleted file mode 100644 index ea8c4bf7f..000000000 --- a/router/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/target diff --git a/router/Cargo.toml b/router/Cargo.toml index 37f319e99..5820c1383 100644 --- a/router/Cargo.toml +++ b/router/Cargo.toml @@ -22,16 +22,7 @@ serde = "1.0.145" serde_json = "1.0.85" thiserror = "1.0.37" tokenizers = "0.13.0" -tokio = { version = "1.21.1", features = ["rt-multi-thread", "parking_lot", "sync"] } +tokio = { version = "1.21.1", features = ["rt", "rt-multi-thread", "parking_lot", "signal", "sync"] } tracing = "0.1.36" tracing-subscriber = "0.3.15" -[workspace] -members = [ - "client", -] - -[profile.release] -debug = 1 -incremental = true -lto = "off" diff --git a/router/client/Cargo.toml b/router/client/Cargo.toml index 7760c8cbe..633f82a9d 100644 --- a/router/client/Cargo.toml +++ b/router/client/Cargo.toml @@ -5,8 +5,6 @@ edition = "2021" [dependencies] futures = "^0.3" -#grpc-error-details = { path = "../../grpc-error-details" } -#grpc-metadata = { path = "../../grpc-metadata" } prost = "^0.9" thiserror = "^1.0" tokio = { version = "^1.21", features = ["sync"] } diff --git a/router/client/src/client.rs b/router/client/src/client.rs index e7189b892..172d0bf73 100644 --- a/router/client/src/client.rs +++ b/router/client/src/client.rs @@ -1,10 +1,11 @@ +/// Single shard Client use crate::pb::generate::v1::text_generation_service_client::TextGenerationServiceClient; use crate::pb::generate::v1::*; use crate::Result; use tonic::transport::{Channel, Uri}; use tracing::*; -/// BLOOM Inference gRPC client +/// Text Generation Inference gRPC client #[derive(Clone)] pub struct Client { stub: TextGenerationServiceClient, @@ -34,6 +35,7 @@ impl Client { }) } + /// Returns a list of uris or unix sockets of all shards #[instrument(skip(self))] pub async fn service_discovery(&mut self) -> Result> { let request = tonic::Request::new(ServiceDiscoveryRequest {}); @@ -46,6 +48,7 @@ impl Client { .into_inner() .urls .into_iter() + // Remove unix socket prefix .map(|url| match url.strip_prefix("unix://") { None => url, Some(stripped_url) => stripped_url.to_string(), @@ -54,6 +57,7 @@ impl Client { Ok(urls) } + /// Clear the past generations cache #[instrument(skip(self))] pub async fn clear_cache(&mut self) -> Result<()> { let request = tonic::Request::new(ClearCacheRequest {}); @@ -64,6 +68,10 @@ impl Client { Ok(()) } + /// Generate one token for each request in the given batch + /// + /// Returns a list of generated texts of request that met their stopping criteria + /// and the next cached batch #[instrument(skip(self))] pub async fn generate(&mut self, batch: Batch) -> Result<(Vec, Option)> { let request = tonic::Request::new(GenerateRequest { batch: Some(batch) }); @@ -76,6 +84,10 @@ impl Client { Ok((response.generated_texts, response.batch)) } + /// Generate one token for each request in the given cached batch + /// + /// Returns a list of generated texts of request that met their stopping criteria + /// and the next cached batch #[instrument(skip(self))] pub async fn generate_with_cache( &mut self, @@ -90,34 +102,4 @@ impl Client { .into_inner(); Ok((response.generated_texts, response.batch)) } - - #[instrument(skip(self))] - pub async fn generate_until_finished( - &mut self, - batch: Batch, - ) -> Result<(Vec, Option)> { - let request = tonic::Request::new(GenerateUntilFinishedRequest { batch: Some(batch) }); - let response = self - .stub - .generate_until_finished(request) - .instrument(info_span!("generate_until_finished")) - .await? - .into_inner(); - Ok((response.generated_texts, response.batch)) - } - - #[instrument(skip(self))] - pub async fn generate_until_finished_with_cache( - &mut self, - batches: Vec, - ) -> Result<(Vec, Option)> { - let request = tonic::Request::new(GenerateUntilFinishedWithCacheRequest { batches }); - let response = self - .stub - .generate_until_finished_with_cache(request) - .instrument(info_span!("generate_until_finished_with_cache")) - .await? - .into_inner(); - Ok((response.generated_texts, response.batch)) - } } diff --git a/router/client/src/lib.rs b/router/client/src/lib.rs index 48b2650d0..0f1f96bca 100644 --- a/router/client/src/lib.rs +++ b/router/client/src/lib.rs @@ -1,6 +1,7 @@ -//! BLOOM Inference gRPC client library +//! Text Generation gRPC client library mod client; +#[allow(clippy::derive_partial_eq_without_eq)] mod pb; mod sharded_client; @@ -8,7 +9,7 @@ pub use client::Client; pub use pb::generate::v1::{Batch, GeneratedText, LogitsWarperParameters, Request}; pub use sharded_client::ShardedClient; use thiserror::Error; -pub use tonic::transport; +use tonic::transport; use tonic::Status; #[derive(Error, Debug, Clone)] @@ -21,7 +22,7 @@ pub enum ClientError { impl From for ClientError { fn from(err: Status) -> Self { - Self::Generation(err.to_string()) + Self::Generation(err.message().to_string()) } } diff --git a/router/client/src/sharded_client.rs b/router/client/src/sharded_client.rs index 7134551e6..916e72b48 100644 --- a/router/client/src/sharded_client.rs +++ b/router/client/src/sharded_client.rs @@ -1,9 +1,11 @@ +/// Multi shard Client use crate::Result; use crate::{Batch, Client, GeneratedText}; use futures::future::join_all; use tokio::sync::{broadcast, mpsc}; use tonic::transport::Uri; +/// List of all available commands that can be sent through the command channel #[derive(Clone, Debug)] enum Command { Generate( @@ -14,36 +16,32 @@ enum Command { Vec, mpsc::Sender, Option)>>, ), - GenerateUntilFinished( - Batch, - mpsc::Sender, Option)>>, - ), - GenerateUntilFinishedWithCache( - Vec, - mpsc::Sender, Option)>>, - ), ClearCache(mpsc::Sender>), } +/// Tokio task that handles the communication with a single shard +/// +/// We subscribe on a broadcast channel to receive commands that will be sent by +/// the ShardedClient. +/// +/// Each command is fan out to all shards. +/// +/// The result of the command is sent back to the ShardedClient through a mpsc channel (multi +/// producer = the shards, single consumer = the ShardedClient). async fn client_task(mut client: Client, mut request_subscriber: broadcast::Receiver) { while let Ok(message) = request_subscriber.recv().await { match message { Command::Generate(batch, response_tx) => { let result = client.generate(batch).await; + // We can unwrap_or(()) here because the only error that can happen is if the + // receiver is dropped, which means that the ShardedClient already received a + // response from another shard response_tx.try_send(result).unwrap_or(()); } Command::GenerateWithCache(batches, response_tx) => { let result = client.generate_with_cache(batches).await; response_tx.try_send(result).unwrap_or(()); } - Command::GenerateUntilFinished(batch, response_tx) => { - let result = client.generate_until_finished(batch).await; - response_tx.try_send(result).unwrap_or(()); - } - Command::GenerateUntilFinishedWithCache(batches, response_tx) => { - let result = client.generate_until_finished_with_cache(batches).await; - response_tx.try_send(result).unwrap_or(()); - } Command::ClearCache(response_tx) => { let result = client.clear_cache().await; response_tx.try_send(result).unwrap_or(()); @@ -52,30 +50,42 @@ async fn client_task(mut client: Client, mut request_subscriber: broadcast::Rece } } +/// Text Generation Inference gRPC multi client pub struct ShardedClient { + _clients: Vec, request_tx: broadcast::Sender, } impl ShardedClient { - fn new(mut clients: Vec) -> Self { + fn new(clients: Vec) -> Self { + // The broadcast channel to communicate with the shards + // We use a capacity of one as the shards are not asynchronous and can only process one + // command at a time let (request_tx, _) = broadcast::channel(1); - for client in clients.drain(..) { + // Spawn client tasks + for client in clients.iter() { let request_subscriber = request_tx.subscribe(); - tokio::spawn(client_task(client, request_subscriber)); + tokio::spawn(client_task(client.clone(), request_subscriber)); } - Self { request_tx } + Self { + _clients: clients, + request_tx, + } } + /// Create a new ShardedClient from a master client. The master client will communicate with + /// the other shards and returns all uris/unix sockets with the `service_discovery` gRPC method. async fn from_master_client(mut master_client: Client) -> Result { + // Get all uris/unix sockets from the master client let uris = master_client.service_discovery().await.unwrap(); - let futures = uris.into_iter().map(|path| Client::connect_uds(path)); + let futures = uris.into_iter().map(Client::connect_uds); let clients: Result> = join_all(futures).await.into_iter().collect(); Ok(Self::new(clients?)) } - /// Returns a client connected to the given url + /// Returns a client connected to the given uri pub async fn connect(uri: Uri) -> Result { let master_client = Client::connect(uri).await?; Self::from_master_client(master_client).await @@ -87,51 +97,43 @@ impl ShardedClient { Self::from_master_client(master_client).await } + /// Generate one token for each request in the given batch + /// + /// Returns a list of generated texts of request that met their stopping criteria + /// and the next cached batch pub async fn generate(&self, batch: Batch) -> Result<(Vec, Option)> { + // Create a channel to receive the response from the shards + // We will only ever receive one message on this channel let (response_tx, mut response_rx) = mpsc::channel(1); self.request_tx .send(Command::Generate(batch, response_tx)) .unwrap(); + // As soon as we receive one response, we can return as all shards will return the same response_rx.recv().await.unwrap() } + /// Generate one token for each request in the given cached batch + /// + /// Returns a list of generated texts of request that met their stopping criteria + /// and the next cached batch pub async fn generate_with_cache( &self, batches: Vec, ) -> Result<(Vec, Option)> { + // Create a channel to receive the response from the shards + // We will only ever receive one message on this channel let (response_tx, mut response_rx) = mpsc::channel(1); self.request_tx .send(Command::GenerateWithCache(batches, response_tx)) .unwrap(); + // As soon as we receive one response, we can return as all shards will return the same response_rx.recv().await.unwrap() } - pub async fn generate_until_finished( - &self, - batch: Batch, - ) -> Result<(Vec, Option)> { - let (response_tx, mut response_rx) = mpsc::channel(1); - self.request_tx - .send(Command::GenerateUntilFinished(batch, response_tx)) - .unwrap(); - response_rx.recv().await.unwrap() - } - - pub async fn generate_until_finished_with_cache( - &self, - batches: Vec, - ) -> Result<(Vec, Option)> { - let (response_tx, mut response_rx) = mpsc::channel(1); - self.request_tx - .send(Command::GenerateUntilFinishedWithCache( - batches, - response_tx, - )) - .unwrap(); - response_rx.recv().await.unwrap() - } - + /// Clear the past generations cache pub async fn clear_cache(&self) -> Result<()> { + // Create a channel to receive the response from the shards + // We will only ever receive one message on this channel let (response_tx, mut response_rx) = mpsc::channel(1); self.request_tx .send(Command::ClearCache(response_tx)) diff --git a/router/src/batcher.rs b/router/src/batcher.rs index ebd817300..4523dbfff 100644 --- a/router/src/batcher.rs +++ b/router/src/batcher.rs @@ -1,129 +1,158 @@ -use crate::server::GenerateRequest; +/// Batching and inference logic +use crate::GenerateRequest; use crate::{Db, Entry}; use axum::http::StatusCode; use bloom_inference_client::{Batch, ClientError, GeneratedText, ShardedClient}; use std::future::Future; use std::sync::Arc; +use std::time::Duration; use thiserror::Error; use tokio::sync::{oneshot, Notify}; +use tokio::time::Instant; +use tracing::instrument; -const MAX_LENGTH: usize = 128; - -#[derive(Debug, Error)] -pub enum InferError { - #[error("Request failed during generation: {0}")] - GenerationError(String), - #[error("Model is overloaded")] - Overloaded, -} - -impl From for (StatusCode, String) { - fn from(err: InferError) -> Self { - match err { - InferError::GenerationError(_) => (StatusCode::INTERNAL_SERVER_ERROR, err.to_string()), - InferError::Overloaded => (StatusCode::TOO_MANY_REQUESTS, err.to_string()), - } - } -} - +/// Batcher #[derive(Clone)] pub struct Batcher { + /// Request database db: Db, + /// Shared state shared: Arc, } +/// Batcher shared state struct Shared { + /// Batching background Tokio task notifier batching_task: Notify, } impl Batcher { - pub(crate) fn new(client: ShardedClient, max_batch_size: usize) -> Self { + pub(crate) fn new( + client: ShardedClient, + max_batch_size: usize, + max_waiting_time: Duration, + ) -> Self { + // Batcher shared state let db = Db::new(); let shared = Arc::new(Shared { batching_task: Notify::new(), }); - tokio::spawn(batching_task(max_batch_size, client, db.clone(), shared.clone())); + // Spawn batching background task that contains all the inference logic + tokio::spawn(batching_task( + max_batch_size, + max_waiting_time, + client, + db.clone(), + shared.clone(), + )); Self { db, shared } } + /// Add a new request to the database and return a future that will generate the text pub(crate) async fn infer( &self, input_length: usize, request: GenerateRequest, ) -> Result { - if self.db.len() > MAX_LENGTH { - return Err(InferError::Overloaded); - } - let (request_tx, request_rx) = oneshot::channel(); + // One shot channel to communicate with the background batching task + let (response_tx, response_rx) = oneshot::channel(); + + // Try to append the request to the database self.db.append(Entry { request, - response_tx: request_tx, + response_tx, input_length, + time: Instant::now(), }); + + // Notify the background task that we have a new entry in the database that needs + // to be batched self.shared.batching_task.notify_waiters(); - match request_rx.await.unwrap() { + + // Await on the response from the background task + // We can safely unwrap as the background task will never drop the sender + match response_rx.await.unwrap() { Ok(output) => Ok(output), Err(err) => Err(InferError::GenerationError(err.to_string())), } } } -async fn batching_task(max_batch_size: usize, - client: ShardedClient, - db: Db, - shared: Arc) { +/// Batching logic +/// Will be launched in a background Tokio task +/// +/// Batches requests and sends them to the inference server +#[instrument(skip(client, db, shared))] +async fn batching_task( + max_batch_size: usize, + max_waiting_time: Duration, + client: ShardedClient, + db: Db, + shared: Arc, +) { + // Minimum batch size after which we try to add more requests let limit_min_batch_size = (max_batch_size / 2) as u32; + // Infinite loop loop { + // Wait for a notification from the Batcher struct shared.batching_task.notified().await; - if let Some(batch) = db.next_batch(max_batch_size) { - let request_ids = batch.requests.iter().map(|req| req.id).collect(); - let mut cached_batch = match batch.size { - size if size > limit_min_batch_size => { - wrap_future(client.generate_until_finished(batch), request_ids, &db).await - } - _ => wrap_future(client.generate(batch), request_ids, &db).await, - }; + // Get the next batch from the DB + // This batch might be smaller than the maximum batch size if there are not enough requests + // waiting in the DB + if let Some((request_ids, batch)) = db.next_batch(None, max_batch_size, None) { + let mut cached_batch = wrap_future(client.generate(batch), request_ids, &db).await; + // We loop until we do not receive any cached batch from the inference server (== until + // all requests have met their stopping criteria) while let Some(batch) = cached_batch { - let mut current_batch_size = batch.size; + // Get current batch info + let batch_size = batch.size; let mut request_ids: Vec = batch.requests.iter().map(|req| req.id).collect(); let mut batches = vec![batch]; - if current_batch_size <= limit_min_batch_size { - if let Some(new_batch) = db.next_batch_minimum_size(limit_min_batch_size as usize, max_batch_size) { - let new_batch_request_ids = - new_batch.requests.iter().map(|req| req.id).collect(); + // If the current batch is too small, we try to add more requests to it + if batch_size <= limit_min_batch_size { + // Get the next batch from the DB that meet our minimum size criteria + if let Some((new_request_ids, new_batch)) = + db.next_batch(Some(limit_min_batch_size as usize), max_batch_size, None) + { + // Generate one token for this new batch to have the attention past in cache let new_cached_batch = - wrap_future(client.generate(new_batch), new_batch_request_ids, &db) - .await; + wrap_future(client.generate(new_batch), new_request_ids, &db).await; + // Extend current batch with the new batch + if let Some(new_cached_batch) = new_cached_batch { + request_ids.extend(new_cached_batch.requests.iter().map(|req| req.id)); + batches.push(new_cached_batch); + } + } + // If we don't have enough requests to meet the minimum size criteria, we + // try to get the next batch from the DB that have been waiting over + // the max_waiting_time + else if let Some((new_request_ids, new_batch)) = + db.next_batch(None, max_batch_size, Some(max_waiting_time)) + { + let new_cached_batch = + wrap_future(client.generate(new_batch), new_request_ids, &db).await; + // Extend current batch with the new batch if let Some(new_cached_batch) = new_cached_batch { - current_batch_size += new_cached_batch.size; request_ids.extend(new_cached_batch.requests.iter().map(|req| req.id)); batches.push(new_cached_batch); } } } - cached_batch = match current_batch_size { - size if size > limit_min_batch_size => { - wrap_future( - client.generate_until_finished_with_cache(batches), - request_ids, - &db, - ) - .await - } - _ => wrap_future(client.generate_with_cache(batches), request_ids, &db).await, - }; + cached_batch = + wrap_future(client.generate_with_cache(batches), request_ids, &db).await; } } } } +/// Wrap a future inside a match statement to handle errors and send the response to the Batcher async fn wrap_future( future: impl Future, Option), ClientError>>, request_ids: Vec, @@ -134,6 +163,7 @@ async fn wrap_future( send_generated(generated_texts, db); next_batch } + // If we have an error, we discard the whole batch Err(err) => { send_error(err, request_ids, db); None @@ -141,16 +171,20 @@ async fn wrap_future( } } +/// Send errors to the Batcher for all `request_ids` fn send_error(error: ClientError, request_ids: Vec, db: &Db) { request_ids.into_iter().for_each(|id| { + // We can `expect` here as the request id should always be in the DB let entry = db.remove(&id).expect("ID not found in db. This is a bug."); // unwrap_or is valid here as we don't care if the receiver is gone. entry.response_tx.send(Err(error.clone())).unwrap_or(()); }); } +/// Send `generated_text` to the Batcher for all `finished` fn send_generated(finished: Vec, db: &Db) { finished.into_iter().for_each(|output| { + // We can `expect` here as the request id should always be in the DB let entry = db .remove(&output.request.unwrap().id) .expect("ID not found in db. This is a bug."); @@ -158,3 +192,18 @@ fn send_generated(finished: Vec, db: &Db) { entry.response_tx.send(Ok(output.output)).unwrap_or(()); }); } + +#[derive(Debug, Error)] +pub enum InferError { + #[error("Request failed during generation: {0}")] + GenerationError(String), +} + +/// Convert to Axum supported format +impl From for (StatusCode, String) { + fn from(err: InferError) -> Self { + match err { + InferError::GenerationError(_) => (StatusCode::INTERNAL_SERVER_ERROR, err.to_string()), + } + } +} diff --git a/router/src/db.rs b/router/src/db.rs index 03593fc07..9518fa1de 100644 --- a/router/src/db.rs +++ b/router/src/db.rs @@ -1,16 +1,173 @@ /// This code is massively inspired by Tokio mini-redis -use crate::server::{GenerateParameters, GenerateRequest}; +use crate::{GenerateParameters, GenerateRequest}; use bloom_inference_client::{Batch, ClientError, LogitsWarperParameters, Request}; -use parking_lot::RwLock; +use parking_lot::Mutex; use std::collections::BTreeMap; use std::sync::Arc; +use std::time::Duration; use tokio::sync::oneshot::Sender; +use tokio::time::Instant; +/// Database entry #[derive(Debug)] pub(crate) struct Entry { + /// Request pub request: GenerateRequest, + /// Response sender to communicate between the Batcher and the batching_task pub response_tx: Sender>, + /// Number of tokens in the input pub input_length: usize, + /// Instant when this entry was created + pub time: Instant, +} + +/// Request Database +#[derive(Debug, Clone)] +pub(crate) struct Db { + pub shared: Arc, +} + +/// Shared state +#[derive(Debug)] +pub struct Shared { + state: Mutex, +} + +/// Database State +#[derive(Debug)] +struct State { + /// Database entries organized in a BTreeMap to be able to iterate over them in order + entries: BTreeMap, + + /// Id of the next entry + next_id: u64, + + /// Id of the next batch + next_batch_id: u64, + + /// Start ID of the next batch. Used to iterate inside the entries BTreeMap + next_batch_start_id: u64, +} + +impl State { + /// Get the next requests + fn next_requests( + &self, + max_size: usize, + min_waiting_time: Option, + ) -> Option<(Vec, Vec)> { + // Iterates for max_size over the BTreemap starting from next_batch_start_id + let mut requests = Vec::new(); + let mut ids = Vec::new(); + + for (id, entry) in self + .entries + // Start from next_batch_start_id + .range(self.next_batch_start_id..) + // Take max_size + .take(max_size) + { + if let Some(min_waiting_time) = min_waiting_time { + // Only take entries that waited for at least min_waiting_time + if entry.time.elapsed() < min_waiting_time { + // Since entries are ordered, we already know that all following entries won't + // satisfy the condition + break; + } + } + + requests.push(Request { + id: *id, + inputs: entry.request.inputs.clone(), + input_length: entry.input_length as u32, + parameters: Some(LogitsWarperParameters::from( + entry.request.parameters.clone(), + )), + max_new_tokens: entry.request.parameters.max_new_tokens, + }); + + ids.push(*id); + } + + if requests.is_empty() { + None + } else { + Some((ids, requests)) + } + } +} + +impl Db { + pub(crate) fn new() -> Self { + // Shared state + let shared = Arc::new(Shared { + state: Mutex::new(State { + entries: BTreeMap::new(), + next_id: 0, + next_batch_id: 0, + next_batch_start_id: 0, + }), + }); + + Self { shared } + } + + /// Append an entry to the database + pub(crate) fn append(&self, entry: Entry) { + // Acquire lock + let mut state = self.shared.state.lock(); + + // Insert entry + let id = state.next_id; + state.next_id += 1; + state.entries.insert(id, entry); + } + + /// Remove an entry from the database if it exists + pub(crate) fn remove(&self, id: &u64) -> Option { + let mut state = self.shared.state.lock(); + state.entries.remove(id) + } + + // Get the next batch + pub(crate) fn next_batch( + &self, + min_size: Option, + max_size: usize, + min_waiting_time: Option, + ) -> Option<(Vec, Batch)> { + // Acquire lock + let mut state = self.shared.state.lock(); + + // Get requests from the database + if let Some((ids, requests)) = state.next_requests(max_size, min_waiting_time) { + if let Some(min_size) = min_size { + // If min_size is set, only return a batch if there are enough requests + if requests.len() < min_size { + return None; + } + } + + // Batch size + let size = requests.len(); + // Longest input length for all requests in batch size + // Used for padding inside the inference server + let max_sequence_length = requests.iter().map(|r| r.input_length).max().unwrap(); + let batch = Batch { + id: state.next_batch_id, + requests, + size: size as u32, + max_sequence_length, + }; + // Update next_batch_start_id to the last id in the batch + 1 + state.next_batch_start_id = ids.last().unwrap() + 1; + // Increment batch id + state.next_batch_id += 1; + + return Some((ids, batch)); + } + None + } } impl From for LogitsWarperParameters { @@ -23,129 +180,3 @@ impl From for LogitsWarperParameters { } } } - -#[derive(Debug, Clone)] -pub(crate) struct Db { - pub shared: Arc, -} - -#[derive(Debug)] -pub struct Shared { - state: RwLock, -} - -#[derive(Debug)] -struct State { - entries: BTreeMap, - - /// Identifier to use for the next expiration. Each expiration is associated - /// with a unique identifier. See above for why. - next_id: u64, - - next_batch_id: u64, - - /// Current batch id - next_batch_start_id: u64, -} - -impl Db { - pub(crate) fn new() -> Self { - let shared = Arc::new(Shared { - state: RwLock::new(State { - entries: BTreeMap::new(), - next_id: 0, - next_batch_id: 0, - next_batch_start_id: 0, - }), - }); - - Self { shared } - } - - pub(crate) fn append(&self, entry: Entry) { - let mut state = self.shared.state.write(); - - let id = state.next_id; - state.next_id += 1; - - state.entries.insert(id, entry); - } - - pub(crate) fn remove(&self, id: &u64) -> Option { - let mut state = self.shared.state.write(); - state.entries.remove(id) - } - - pub(crate) fn len(&self) -> usize { - let state = self.shared.state.read(); - state.entries.len() - } - - fn next_requests(&self, max_size: usize) -> Option<(u64, Vec)> { - let state = self.shared.state.read(); - - let requests: Vec = state - .entries - .range(state.next_batch_start_id..) - .take(max_size) - .map(|(id, entry)| Request { - id: *id, - inputs: entry.request.inputs.clone(), - input_length: entry.input_length as u32, - parameters: Some(LogitsWarperParameters::from( - entry.request.parameters.clone(), - )), - max_new_tokens: entry.request.parameters.max_new_tokens, - }) - .collect(); - - if requests.is_empty() { - None - } else { - let last_id = requests.last().unwrap().id; - Some((last_id, requests)) - } - } - - pub(crate) fn next_batch(&self, max_size: usize) -> Option { - if let Some((last_id, requests)) = self.next_requests(max_size) { - let mut state = self.shared.state.write(); - let size = requests.len(); - let max_sequence_length = requests.iter().map(|r| r.input_length).max().unwrap(); - let batch = Batch { - id: state.next_batch_id, - requests, - size: size as u32, - max_sequence_length, - }; - state.next_batch_start_id = last_id + 1; - state.next_batch_id += 1; - return Some(batch); - } - None - } - - pub(crate) fn next_batch_minimum_size( - &self, - min_size: usize, - max_size: usize, - ) -> Option { - if let Some((last_id, requests)) = self.next_requests(max_size) { - if requests.len() >= min_size { - let mut state = self.shared.state.write(); - let size = requests.len(); - let max_sequence_length = requests.iter().map(|r| r.input_length).max().unwrap(); - let batch = Batch { - id: state.next_batch_id, - requests, - size: size as u32, - max_sequence_length, - }; - state.next_batch_start_id = last_id + 1; - state.next_batch_id += 1; - return Some(batch); - } - } - None - } -} diff --git a/router/src/lib.rs b/router/src/lib.rs index 14dc57249..02b912a3e 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -1,8 +1,68 @@ +/// Text Generation Inference Webserver mod batcher; mod db; -mod validation; pub mod server; +mod validation; -use db::{Db, Entry}; use batcher::Batcher; +use db::{Db, Entry}; +use serde::{Deserialize, Serialize}; use validation::Validation; + +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct GenerateParameters { + #[serde(default = "default_temperature")] + pub temperature: f32, + #[serde(default = "default_top_k")] + pub top_k: i32, + #[serde(default = "default_top_p")] + pub top_p: f32, + #[serde(default = "default_do_sample")] + pub do_sample: bool, + #[serde(default = "default_max_new_tokens")] + pub max_new_tokens: u32, +} + +fn default_temperature() -> f32 { + 1.0 +} + +fn default_top_k() -> i32 { + 0 +} + +fn default_top_p() -> f32 { + 1.0 +} + +fn default_do_sample() -> bool { + false +} + +fn default_max_new_tokens() -> u32 { + 20 +} + +fn default_parameters() -> GenerateParameters { + GenerateParameters { + temperature: default_temperature(), + top_k: default_top_k(), + top_p: default_top_p(), + do_sample: default_do_sample(), + max_new_tokens: default_max_new_tokens(), + } +} + +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct GenerateRequest { + pub inputs: String, + #[serde(default = "default_parameters")] + pub parameters: GenerateParameters, +} + +#[derive(Serialize)] +pub(crate) struct GeneratedText { + pub generated_text: String, +} + +pub(crate) type GenerateResponse = Vec; diff --git a/router/src/main.rs b/router/src/main.rs index 89cd47313..49051b376 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -1,37 +1,61 @@ +/// Text Generation Inference webserver entrypoint use bloom_inference_client::ShardedClient; +use clap::Parser; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use std::time::Duration; use text_generation_router::server; use tokenizers::Tokenizer; -use clap::Parser; /// App Configuration #[derive(Parser, Debug)] #[clap(author, version, about, long_about = None)] struct Args { - #[clap(default_value = "32", long, short, env)] + #[clap(default_value = "128", long, env)] + max_concurrent_requests: usize, + #[clap(default_value = "1000", long, env)] + max_input_length: usize, + #[clap(default_value = "32", long, env)] max_batch_size: usize, + #[clap(default_value = "5", long, env)] + max_waiting_time: u64, #[clap(default_value = "3000", long, short, env)] port: u16, #[clap(default_value = "/tmp/bloom-inference-0", long, env)] - shard_uds_path: String, + master_shard_uds_path: String, #[clap(default_value = "bigscience/bloom", long, env)] tokenizer_name: String, + #[clap(default_value = "2", long, env)] + validation_workers: usize, } fn main() -> Result<(), std::io::Error> { // Get args let args = Args::parse(); -// Pattern match configuration + // Pattern match configuration let Args { + max_concurrent_requests, + max_input_length, max_batch_size, + max_waiting_time, port, - shard_uds_path, + master_shard_uds_path, tokenizer_name, + validation_workers, } = args; + if validation_workers == 1 { + panic!("validation_workers must be > 0"); + } + let max_waiting_time = Duration::from_secs(max_waiting_time); + + // Download and instantiate tokenizer + // This will only be used to validate payloads + // + // We need to download it outside of the Tokio runtime let tokenizer = Tokenizer::from_pretrained(tokenizer_name, None).unwrap(); + // Launch Tokio runtime tokio::runtime::Builder::new_multi_thread() .enable_all() .build() @@ -39,18 +63,32 @@ fn main() -> Result<(), std::io::Error> { .block_on(async { tracing_subscriber::fmt::init(); - let sharded_client = ShardedClient::connect_uds(shard_uds_path) + // Instantiate sharded client from the master unix socket + let sharded_client = ShardedClient::connect_uds(master_shard_uds_path) .await .expect("Could not connect to server"); + // Clear the cache; useful if the webserver rebooted sharded_client .clear_cache() .await .expect("Unable to clear cache"); tracing::info!("Connected"); + // Binds on localhost let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), port); - server::run(max_batch_size, sharded_client, tokenizer, addr).await; + // Run server + server::run( + max_concurrent_requests, + max_input_length, + max_batch_size, + max_waiting_time, + sharded_client, + tokenizer, + validation_workers, + addr, + ) + .await; Ok(()) }) } diff --git a/router/src/server.rs b/router/src/server.rs index 0fdfd58b9..422583135 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -1,68 +1,44 @@ -use crate::{Batcher, Validation}; +use crate::{ + Batcher, GenerateParameters, GenerateRequest, GenerateResponse, GeneratedText, Validation, +}; use axum::extract::Extension; use axum::http::StatusCode; use axum::routing::{get, post}; use axum::{Json, Router}; use bloom_inference_client::ShardedClient; -use serde::Deserialize; use std::net::SocketAddr; +use std::sync::Arc; +use std::time::Duration; use tokenizers::Tokenizer; +use tokio::signal; +use tokio::sync::Semaphore; use tokio::time::Instant; use tracing::instrument; -#[derive(Clone, Debug, Deserialize)] -pub(crate) struct GenerateParameters { - #[serde(default = "default_temperature")] - pub temperature: f32, - #[serde(default = "default_top_k")] - pub top_k: i32, - #[serde(default = "default_top_p")] - pub top_p: f32, - #[serde(default = "default_do_sample")] - pub do_sample: bool, - #[serde(default = "default_max_new_tokens")] - pub max_new_tokens: u32, -} - -fn default_temperature() -> f32 { - 1.0 -} - -fn default_top_k() -> i32 { - 0 -} - -fn default_top_p() -> f32 { - 1.0 -} - -fn default_do_sample() -> bool { - false -} - -fn default_max_new_tokens() -> u32 { - 20 -} - -fn default_parameters() -> GenerateParameters { - GenerateParameters { - temperature: default_temperature(), - top_k: default_top_k(), - top_p: default_top_p(), - do_sample: default_do_sample(), - max_new_tokens: default_max_new_tokens(), - } -} - -#[derive(Clone, Debug, Deserialize)] -pub(crate) struct GenerateRequest { - pub inputs: String, - #[serde(default = "default_parameters")] - pub parameters: GenerateParameters, +// Server shared state +#[derive(Clone)] +struct ServerState { + validation: Validation, + batcher: Batcher, + limit_concurrent_requests: Arc, } +/// Health check method #[instrument(skip(state), fields(time, time_per_token))] -async fn liveness(state: Extension) -> Result<(), (StatusCode, String)> { +async fn health(state: Extension) -> Result<(), (StatusCode, String)> { + // TODO: while this is the best health check we can do, it is a bit on the heavy side and might + // be a bit too slow for a health check. + // What we should do instead if check if the gRPC channels are still healthy. + + // Limit concurrent requests by acquiring a permit from the semaphore + let _permit = state.limit_concurrent_requests.try_acquire().map_err(|_| { + ( + StatusCode::TOO_MANY_REQUESTS, + "Model is overloaded".to_string(), + ) + })?; + + // Send a small inference request state .batcher .infer( @@ -82,23 +58,35 @@ async fn liveness(state: Extension) -> Result<(), (StatusCode, Stri Ok(()) } +/// Generate method #[instrument(skip(state), fields(time, time_per_token))] async fn generate( state: Extension, req: Json, -) -> Result, (StatusCode, String)> { +) -> Result, (StatusCode, String)> { let start = Instant::now(); + // Limit concurrent requests by acquiring a permit from the semaphore + let _permit = state.limit_concurrent_requests.try_acquire().map_err(|_| { + ( + StatusCode::TOO_MANY_REQUESTS, + "Model is overloaded".to_string(), + ) + })?; + // Validate request let (input_length, validated_request) = state .validation + // FIXME: can't we get rid of the cloning here?? .validate(GenerateRequest { inputs: req.inputs.clone(), parameters: req.parameters.clone(), }) .await?; + // Inference let generated_text = state.batcher.infer(input_length, validated_request).await?; + // Tracing metadata tracing::Span::current().record("time", format!("{:?}", start.elapsed())); tracing::Span::current().record( "time_per_token", @@ -106,31 +94,71 @@ async fn generate( ); tracing::info!("response: {}", generated_text); - Ok(Json(serde_json::json!({ - "generated_text": generated_text, - }))) + // Send response + let response = vec![GeneratedText { generated_text }]; + Ok(Json(response)) } -#[derive(Clone)] -struct ServerState { - validation: Validation, - batcher: Batcher, -} - -pub async fn run(max_batch_size: usize, client: ShardedClient, tokenizer: Tokenizer, addr: SocketAddr) { - let batcher = Batcher::new(client, max_batch_size); - let validation = Validation::new(tokenizer); - - let shared_state = ServerState { validation, batcher }; +/// Serving method +#[allow(clippy::too_many_arguments)] +pub async fn run( + max_concurrent_requests: usize, + max_input_length: usize, + max_batch_size: usize, + max_waiting_time: Duration, + client: ShardedClient, + tokenizer: Tokenizer, + validation_workers: usize, + addr: SocketAddr, +) { + // Create state + let batcher = Batcher::new(client, max_batch_size, max_waiting_time); + let validation = Validation::new(validation_workers, tokenizer, max_input_length); + let shared_state = ServerState { + validation, + batcher, + limit_concurrent_requests: Arc::new(Semaphore::new(max_concurrent_requests)), + }; + // Create router let app = Router::new() .route("/generate", post(generate)) .layer(Extension(shared_state.clone())) - .route("/health", get(liveness)) + .route("/health", get(health)) .layer(Extension(shared_state.clone())); + // Run server axum::Server::bind(&addr) .serve(app.into_make_service()) + // Wait until all requests are finished to shut down + .with_graceful_shutdown(shutdown_signal()) .await .unwrap(); } + +/// Shutdown signal handler +async fn shutdown_signal() { + let ctrl_c = async { + signal::ctrl_c() + .await + .expect("failed to install Ctrl+C handler"); + }; + + #[cfg(unix)] + let terminate = async { + signal::unix::signal(signal::unix::SignalKind::terminate()) + .expect("failed to install signal handler") + .recv() + .await; + }; + + #[cfg(not(unix))] + let terminate = std::future::pending::<()>(); + + tokio::select! { + _ = ctrl_c => {}, + _ = terminate => {}, + } + + tracing::info!("signal received, starting graceful shutdown"); +} diff --git a/router/src/validation.rs b/router/src/validation.rs index 45b108fd4..49a46b624 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -1,62 +1,105 @@ -use crate::server::GenerateRequest; +/// Payload validation logic +use crate::GenerateRequest; use axum::http::StatusCode; use thiserror::Error; use tokenizers::tokenizer::Tokenizer; +use tokenizers::{ + DecoderWrapper, ModelWrapper, NormalizerWrapper, PostProcessorWrapper, PreTokenizerWrapper, + TokenizerImpl, +}; use tokio::sync::{mpsc, oneshot}; -#[derive(Error, Debug)] -pub enum ValidationError { - #[error("Temperature must be strictly positive")] - Temperature, - #[error("Top p must be <= 0.0 or > 1.0")] - TopP, - #[error("Top k must be strictly positive")] - TopK, - #[error("Max New Tokens must be < 512")] - MaxNewTokens, - #[error("Inputs must have less than 1000 tokens. Given: {0}")] - InputLength(usize), -} - -impl From for (StatusCode, String) { - fn from(err: ValidationError) -> Self { - (StatusCode::BAD_REQUEST, err.to_string()) - } -} - -type ValidationRequest = ( - GenerateRequest, - oneshot::Sender>, -); - +/// Validation #[derive(Debug, Clone)] pub struct Validation { + /// Channel to communicate with the background validation task sender: mpsc::Sender, } impl Validation { - pub(crate) fn new(tokenizer: Tokenizer) -> Self { + pub(crate) fn new(workers: usize, tokenizer: Tokenizer, max_input_length: usize) -> Self { + // Crate channel let (validation_sender, validation_receiver) = mpsc::channel(128); - tokio::spawn(validation_task(tokenizer, validation_receiver)); + // Launch background validation task + tokio::spawn(validation_task( + workers, + tokenizer, + max_input_length, + validation_receiver, + )); Self { sender: validation_sender, } } + /// Validate a payload and get the number of tokens in the input pub(crate) async fn validate( &self, request: GenerateRequest, ) -> Result<(usize, GenerateRequest), ValidationError> { + // Create response channel let (sender, receiver) = oneshot::channel(); + // Send request to the background validation task + // Unwrap is safe here self.sender.send((request, sender)).await.unwrap(); + // Await on response channel + // Unwrap is safe here receiver.await.unwrap() } } -async fn validation_task(tokenizer: Tokenizer, mut receiver: mpsc::Receiver) { - while let Some((request, response_tx)) = receiver.recv().await { +/// Validation task +/// Load balance the validation requests between multiple validation workers +async fn validation_task( + workers: usize, + tokenizer: Tokenizer, + max_input_length: usize, + mut receiver: mpsc::Receiver, +) { + let mut workers_senders = Vec::with_capacity(workers); + + // Create workers + for _ in 0..workers { + let tokenizer_clone = tokenizer.clone(); + // Create channel to communicate with worker + let (worker_sender, worker_receiver) = mpsc::channel(workers); + workers_senders.push(worker_sender); + + // Spawn worker + tokio::task::spawn_blocking(move || { + validation_worker(tokenizer_clone, max_input_length, worker_receiver) + }); + } + + loop { + // Load balance requests between workers + for sender in workers_senders.iter() { + if let Some(validation_request) = receiver.recv().await { + sender.send(validation_request).await.unwrap(); + } else { + return; + } + } + } +} + +/// Check the parameters inside the payload and get the number of tokens inside the input using +/// the tokenizer +fn validation_worker( + tokenizer: TokenizerImpl< + ModelWrapper, + NormalizerWrapper, + PreTokenizerWrapper, + PostProcessorWrapper, + DecoderWrapper, + >, + max_input_length: usize, + mut receiver: mpsc::Receiver, +) { + // Loop over requests + while let Some((request, response_tx)) = receiver.blocking_recv() { if request.parameters.temperature < 0.0 { response_tx .send(Err(ValidationError::Temperature)) @@ -78,10 +121,11 @@ async fn validation_task(tokenizer: Tokenizer, mut receiver: mpsc::Receiver 1000 { + if input_length > max_input_length { response_tx .send(Err(ValidationError::InputLength(input_length))) .unwrap_or(()); @@ -91,3 +135,28 @@ async fn validation_task(tokenizer: Tokenizer, mut receiver: mpsc::Receiver>, +); + +#[derive(Error, Debug)] +pub enum ValidationError { + #[error("Temperature must be strictly positive")] + Temperature, + #[error("Top p must be <= 0.0 or > 1.0")] + TopP, + #[error("Top k must be strictly positive")] + TopK, + #[error("Max New Tokens must be < 512")] + MaxNewTokens, + #[error("Inputs must have less than 1000 tokens. Given: {0}")] + InputLength(usize), +} + +impl From for (StatusCode, String) { + fn from(err: ValidationError) -> Self { + (StatusCode::BAD_REQUEST, err.to_string()) + } +} diff --git a/run.sh b/run.sh deleted file mode 100644 index 303035017..000000000 --- a/run.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash - -server_cmd="bloom-inference-server launcher $MODEL_NAME --num-gpus $NUM_GPUS --shard-directory $MODEL_BASE_PATH" - -# Run in background -$server_cmd 2>&1 > /dev/null & - -# Check if server is running by checking if the unix socket is created -FILE=/tmp/bloom-inference-0 -while : - do - if test -S "$FILE"; then - echo "Text Generation Python gRPC server started" - break - else - echo "Waiting for Text Generation Python gRPC server to start" - sleep 5 - fi - done - -sleep 1 - -# Run in background -text-generation-router & - -# Wait for any process to exit -wait -n - -# Exit with status of process that exited first -exit $? \ No newline at end of file diff --git a/router/rust-toolchain.toml b/rust-toolchain.toml similarity index 100% rename from router/rust-toolchain.toml rename to rust-toolchain.toml diff --git a/server/.gitignore b/server/.gitignore new file mode 100644 index 000000000..0ebf3670b --- /dev/null +++ b/server/.gitignore @@ -0,0 +1,155 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +bloom_inference/__pycache__/ +bloom_inference/pb/__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ diff --git a/server/Makefile b/server/Makefile index d959dbb38..52b4d4059 100644 --- a/server/Makefile +++ b/server/Makefile @@ -4,17 +4,28 @@ gen-server: find bloom_inference/pb/ -type f -name "*.py" -print0 -exec sed -i -e 's/^\(import.*pb2\)/from . \1/g' {} \; touch bloom_inference/pb/__init__.py -unit-tests: - python -m pytest --cov=bloom_inference tests +install-transformers: + # Install specific version of transformers + rm transformers || true + wget https://github.com/huggingface/transformers/archive/46d37bece7d3ffdef97b1ee4a3170c0a0627d921.zip + unzip 46d37bece7d3ffdef97b1ee4a3170c0a0627d921.zip + rm 46d37bece7d3ffdef97b1ee4a3170c0a0627d921.zip + mv transformers-46d37bece7d3ffdef97b1ee4a3170c0a0627d921 transformers + cd transformers && python setup.py install -unit-tests-reporting: - python -m pytest --junitxml=report.xml --cov=bloom_inference tests +install-torch: + # Install specific version of torch + pip install torch --extra-index-url https://download.pytorch.org/whl/cu116 --no-cache-dir pip-install: pip install grpcio-tools make gen-server + make install-torch + make install-transformers pip install . install: poetry install - make gen-server \ No newline at end of file + make gen-server + make install-torch + make install-transformers diff --git a/server/bloom_inference/cli.py b/server/bloom_inference/cli.py index a5f84e77f..751485bb0 100644 --- a/server/bloom_inference/cli.py +++ b/server/bloom_inference/cli.py @@ -1,41 +1,51 @@ +import os import typer from pathlib import Path -from torch.distributed.launcher import launch_agent, LaunchConfig from typing import Optional -from bloom_inference import server +from bloom_inference import prepare_weights, server app = typer.Typer() @app.command() -def launcher( - model_name: str, - num_gpus: int = 1, - shard_directory: Optional[Path] = None, +def serve( + model_name: str, + sharded: bool = False, + shard_directory: Optional[Path] = None, + uds_path: Path = "/tmp/bloom-inference", ): - if num_gpus == 1: - serve(model_name, False, shard_directory) + if sharded: + assert ( + shard_directory is not None + ), "shard_directory must be set when sharded is True" + assert ( + os.getenv("RANK", None) is not None + ), "RANK must be set when sharded is True" + assert ( + os.getenv("WORLD_SIZE", None) is not None + ), "WORLD_SIZE must be set when sharded is True" + assert ( + os.getenv("MASTER_ADDR", None) is not None + ), "MASTER_ADDR must be set when sharded is True" + assert ( + os.getenv("MASTER_PORT", None) is not None + ), "MASTER_PORT must be set when sharded is True" - else: - config = LaunchConfig( - min_nodes=1, - max_nodes=1, - nproc_per_node=num_gpus, - rdzv_backend="c10d", - max_restarts=0, - ) - launch_agent(config, server.serve, [model_name, True, shard_directory]) + server.serve(model_name, sharded, uds_path, shard_directory) @app.command() -def serve( - model_name: str, - sharded: bool = False, - shard_directory: Optional[Path] = None, +def prepare_weights( + model_name: str, + shard_directory: Path, + cache_directory: Path, + num_shard: int = 1, ): - server.serve(model_name, sharded, shard_directory) + prepare_weights.prepare_weights( + model_name, cache_directory, shard_directory, num_shard + ) if __name__ == "__main__": diff --git a/server/bloom_inference/model.py b/server/bloom_inference/model.py index 8b0e7ab0b..0ba90cee3 100644 --- a/server/bloom_inference/model.py +++ b/server/bloom_inference/model.py @@ -24,6 +24,7 @@ torch.manual_seed(0) class Batch: batch_id: int requests: List[generate_pb2.Request] + all_input_lengths: List[int] input_ids: Dict[str, torch.Tensor] all_input_ids: List[torch.Tensor] next_token_choosers: List[NextTokenChooser] @@ -46,12 +47,12 @@ class Batch: inputs = [] next_token_choosers = [] stopping_criterias = [] - input_lengths = [] + all_input_lengths = [] # Parse batch for r in pb.requests: inputs.append(r.inputs) - input_lengths.append(r.input_length) + all_input_lengths.append(r.input_length) next_token_choosers.append( NextTokenChooser( temperature=r.parameters.temperature, @@ -63,17 +64,12 @@ class Batch: stopping_criterias.append(StoppingCriteria(max_new_tokens=r.max_new_tokens)) input_ids = tokenizer(inputs, return_tensors="pt", padding=True).to(device) - # Remove padding from all_input_ids - all_input_ids = [ - input_ids.squeeze(0)[-length:].unsqueeze(-1) - for length, input_ids in zip( - input_lengths, input_ids["input_ids"].split(1, dim=0) - ) - ] + all_input_ids = input_ids["input_ids"].unsqueeze(-1) return cls( batch_id=pb.id, requests=pb.requests, + all_input_lengths=all_input_lengths, input_ids=input_ids, all_input_ids=all_input_ids, next_token_choosers=next_token_choosers, @@ -91,6 +87,7 @@ class Batch: # Batch attributes input_ids = {"input_ids": None, "attention_mask": None, "past_key_values": []} requests = [] + all_input_lengths = [] all_input_ids = [] next_token_choosers = [] stopping_criterias = [] @@ -100,6 +97,7 @@ class Batch: start_index = 0 for i, batch in enumerate(batches): requests.extend(batch.requests) + all_input_lengths.extend(batch.all_input_lengths) all_input_ids.extend(batch.all_input_ids) next_token_choosers.extend(batch.next_token_choosers) stopping_criterias.extend(batch.stopping_criterias) @@ -198,6 +196,7 @@ class Batch: return cls( batch_id=batches[0].batch_id, requests=requests, + all_input_lengths=all_input_lengths, input_ids=input_ids, all_input_ids=all_input_ids, next_token_choosers=next_token_choosers, @@ -227,7 +226,10 @@ class BLOOM: self.tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left") self.model = ( - AutoModelForCausalLM.from_pretrained(model_name).eval().to(self.device).to(dtype) + AutoModelForCausalLM.from_pretrained(model_name) + .eval() + .to(self.device) + .to(dtype) ) self.num_heads = self.model.base_model.num_heads @@ -253,6 +255,7 @@ class BLOOM: # New input_ids for next forward next_batch_input_ids = [] next_batch_all_input_ids = [] + next_all_input_lengths = [] next_batch_size = 0 next_batch_max_sequence_length = 0 @@ -263,6 +266,7 @@ class BLOOM: # Zipped iterator iterator = zip( batch.requests, + batch.all_input_lengths, outputs.logits, batch.next_token_choosers, batch.stopping_criterias, @@ -272,6 +276,7 @@ class BLOOM: # For each member of the batch for i, ( request, + input_length, logits, next_token_chooser, stopping_criteria, @@ -302,8 +307,10 @@ class BLOOM: next_batch_input_ids.append(next_token) next_batch_all_input_ids.append(all_tokens) next_batch_size += 1 + new_input_length = input_length + 1 + next_all_input_lengths.append(new_input_length) next_batch_max_sequence_length = max( - next_batch_max_sequence_length, len(all_tokens) + next_batch_max_sequence_length, new_input_length ) # We finished all generations in the batch; there is no next batch @@ -350,6 +357,7 @@ class BLOOM: next_batch = Batch( batch_id=batch.batch_id, requests=next_batch_requests, + all_input_lengths=next_all_input_lengths, input_ids=next_batch_input_ids, all_input_ids=next_batch_all_input_ids, next_token_choosers=next_batch_next_token_choosers, @@ -378,7 +386,10 @@ class BLOOMSharded(BLOOM): if self.master: # TODO @thomasw21 do some caching shard_state_dict_paths = prepare_weights( - model_name, shard_directory / "cache", shard_directory, tp_world_size=self.world_size + model_name, + shard_directory / "cache", + shard_directory, + tp_world_size=self.world_size, ) shard_state_dict_paths = [ str(path.absolute()) for path in shard_state_dict_paths @@ -443,6 +454,7 @@ class BLOOMSharded(BLOOM): use_cache=True, ) + # Logits are sharded, so we need to gather them logits_shard = outputs.logits[:, -1, :].contiguous() batch_size, vocab_shard_size = logits_shard.shape diff --git a/server/bloom_inference/pb/.gitignore b/server/bloom_inference/pb/.gitignore index a9feac816..8527ad136 100644 --- a/server/bloom_inference/pb/.gitignore +++ b/server/bloom_inference/pb/.gitignore @@ -1,2 +1,2 @@ *.py -*.py-e +*.py-e \ No newline at end of file diff --git a/server/bloom_inference/prepare_weights.py b/server/bloom_inference/prepare_weights.py index 7cf3dbb5c..5594998d0 100644 --- a/server/bloom_inference/prepare_weights.py +++ b/server/bloom_inference/prepare_weights.py @@ -14,15 +14,15 @@ from huggingface_hub.file_download import _request_wrapper, hf_raise_for_status def match_suffix(text, suffix): - return text[-len(suffix):] == suffix + return text[-len(suffix) :] == suffix def http_get( - url: str, - temp_file: BinaryIO, - *, - timeout=10.0, - max_retries=0, + url: str, + temp_file: BinaryIO, + *, + timeout=10.0, + max_retries=0, ): """ Download a remote file. Do not gobble up errors, and will return errors tailored to the Hugging Face Hub. @@ -54,7 +54,9 @@ def cache_download_url(url: str, root_dir: Path): return filename -def prepare_weights(model_name: str, cache_path: Path, save_path: Path, tp_world_size: int): +def prepare_weights( + model_name: str, cache_path: Path, save_path: Path, tp_world_size: int +): save_paths = [ save_path / f"{model_name}_tp-rank-{tp_rank}-of-{tp_world_size}.pty" for tp_rank in range(tp_world_size) @@ -68,6 +70,7 @@ def prepare_weights(model_name: str, cache_path: Path, save_path: Path, tp_world if model_name == "bigscience/bloom-560m": url = hf_hub_url(model_name, filename="pytorch_model.bin") cache_download_url(url, cache_path) + elif model_name == "bigscience/bloom": url = hf_hub_url(model_name, filename="pytorch_model.bin.index.json") index_path = cache_download_url(url, cache_path) @@ -75,10 +78,14 @@ def prepare_weights(model_name: str, cache_path: Path, save_path: Path, tp_world index = json.load(f) # Get unique file names - weight_files = list(set([filename for filename in index["weight_map"].values()])) + weight_files = list( + set([filename for filename in index["weight_map"].values()]) + ) urls = [hf_hub_url(model_name, filename=filename) for filename in weight_files] - Parallel(n_jobs=5)(delayed(cache_download_url)(url, cache_path) for url in tqdm(urls)) + Parallel(n_jobs=5)( + delayed(cache_download_url)(url, cache_path) for url in tqdm(urls) + ) else: raise ValueError(f"Unknown model name: {model_name}") @@ -91,14 +98,14 @@ def prepare_weights(model_name: str, cache_path: Path, save_path: Path, tp_world for state_name in keys: state = state_dict[state_name] if any( - match_suffix(state_name, candidate) - for candidate in [ - "self_attention.query_key_value.weight", - "self_attention.query_key_value.bias", - "mlp.dense_h_to_4h.weight", - "mlp.dense_h_to_4h.bias", - "word_embeddings.weight", - ] + match_suffix(state_name, candidate) + for candidate in [ + "self_attention.query_key_value.weight", + "self_attention.query_key_value.bias", + "mlp.dense_h_to_4h.weight", + "mlp.dense_h_to_4h.bias", + "word_embeddings.weight", + ] ): output_size = state.shape[0] assert output_size % tp_world_size == 0 @@ -107,7 +114,9 @@ def prepare_weights(model_name: str, cache_path: Path, save_path: Path, tp_world assert len(sharded_weights) == tp_world_size for tp_rank, shard in enumerate(sharded_weights): - shards_state_dicts[tp_rank]["transformer." + state_name] = shard.detach().clone() + shards_state_dicts[tp_rank][ + "transformer." + state_name + ] = shard.detach().clone() elif match_suffix(state_name, "lm_head.weight"): output_size = state.shape[0] @@ -120,11 +129,11 @@ def prepare_weights(model_name: str, cache_path: Path, save_path: Path, tp_world shards_state_dicts[tp_rank][state_name] = shard.detach().clone() elif any( - match_suffix(state_name, candidate) - for candidate in [ - "self_attention.dense.weight", - "mlp.dense_4h_to_h.weight", - ] + match_suffix(state_name, candidate) + for candidate in [ + "self_attention.dense.weight", + "mlp.dense_4h_to_h.weight", + ] ): input_size = state.shape[1] assert input_size % tp_world_size == 0 @@ -132,23 +141,31 @@ def prepare_weights(model_name: str, cache_path: Path, save_path: Path, tp_world sharded_weights = torch.split(state, block_size, dim=1) assert len(sharded_weights) == tp_world_size for tp_rank, shard in enumerate(sharded_weights): - shards_state_dicts[tp_rank]["transformer." + state_name] = shard.detach().clone() + shards_state_dicts[tp_rank][ + "transformer." + state_name + ] = shard.detach().clone() elif any( - match_suffix(state_name, candidate) - for candidate in [ - "self_attention.dense.bias", - "mlp.dense_4h_to_h.bias", - ] + match_suffix(state_name, candidate) + for candidate in [ + "self_attention.dense.bias", + "mlp.dense_4h_to_h.bias", + ] ): - shards_state_dicts[0]["transformer." + state_name] = state.detach().clone() + shards_state_dicts[0][ + "transformer." + state_name + ] = state.detach().clone() for tp_rank in range(1, tp_world_size): - shards_state_dicts[tp_rank]["transformer." + state_name] = torch.zeros_like(state) + shards_state_dicts[tp_rank][ + "transformer." + state_name + ] = torch.zeros_like(state) else: # We duplicate parameters across tp ranks for tp_rank in range(tp_world_size): - shards_state_dicts[tp_rank]["transformer." + state_name] = state.detach().clone() + shards_state_dicts[tp_rank][ + "transformer." + state_name + ] = state.detach().clone() del state_dict[state_name] # delete key from state_dict del state # delete tensor @@ -156,7 +173,7 @@ def prepare_weights(model_name: str, cache_path: Path, save_path: Path, tp_world # we save state_dict for tp_rank, (save_path, shard_state_dict) in enumerate( - zip(save_paths, shards_state_dicts) + zip(save_paths, shards_state_dicts) ): save_paths.append(save_path) save_path.parent.mkdir(parents=True, exist_ok=True) @@ -166,17 +183,3 @@ def prepare_weights(model_name: str, cache_path: Path, save_path: Path, tp_world torch.save(shard_state_dict, save_path) return save_paths - - -if __name__ == "__main__": - from argparse import ArgumentParser - - parser = ArgumentParser() - - parser.add_argument("--model-name", required=True, type=str) - parser.add_argument("--cache-path", required=True, type=str) - parser.add_argument("--save-path", required=True, type=str) - parser.add_argument("--world-size", required=True, type=int) - args = parser.parse_args() - - prepare_weights(args.model_name, Path(args.cache_path), Path(args.save_path), args.world_size) diff --git a/server/bloom_inference/server.py b/server/bloom_inference/server.py index e89706e0a..734aba439 100644 --- a/server/bloom_inference/server.py +++ b/server/bloom_inference/server.py @@ -64,70 +64,31 @@ class TextGenerationService(generate_pb2_grpc.TextGenerationServiceServicer): batch=next_batch.to_pb() if next_batch else None, ) - async def GenerateUntilFinished(self, request, context): - batch = Batch.from_pb(request.batch, self.model.tokenizer, self.model.device) - generated_texts = [] - while not generated_texts: - generated_texts, next_batch = self.model.generate_token(batch) - batch = next_batch - self.cache.set(next_batch) - - return generate_pb2.GenerateUntilFinishedResponse( - generated_texts=[ - generated_text.to_pb() for generated_text in generated_texts - ], - batch=next_batch.to_pb() if next_batch else None, - ) - - async def GenerateUntilFinishedWithCache(self, request, context): - if len(request.batches) == 0: - raise ValueError("Must provide at least one batch") - - batches = [] - for batch_pb in request.batches: - batch = self.cache.pop(batch_pb.id) - if batch is None: - raise ValueError(f"Batch ID {batch_pb.id} not found in cache.") - batches.append(batch) - - if len(batches) > 1: - batch = Batch.concatenate(batches) - else: - batch = batches[0] - - generated_texts = [] - while not generated_texts: - generated_texts, next_batch = self.model.generate_token(batch) - batch = next_batch - self.cache.set(next_batch) - - return generate_pb2.GenerateUntilFinishedWithCacheResponse( - generated_texts=[ - generated_text.to_pb() for generated_text in generated_texts - ], - batch=next_batch.to_pb() if next_batch else None, - ) - - -def serve(model_name, sharded, shard_directory): +def serve( + model_name: str, + sharded: bool, + uds_path: Path, + shard_directory: Optional[Path] = None, +): async def serve_inner( model_name: str, sharded: bool = False, shard_directory: Optional[Path] = None, ): - unix_socket_template = "unix:///tmp/bloom-inference-{}" + unix_socket_template = "unix://{}-{}" if sharded: if shard_directory is None: raise ValueError("shard_directory must be set when sharded is True") model = BLOOMSharded(model_name, shard_directory) server_urls = [ - unix_socket_template.format(rank) for rank in range(model.world_size) + unix_socket_template.format(uds_path, rank) + for rank in range(model.world_size) ] - local_url = unix_socket_template.format(model.rank) + local_url = server_urls[model.rank] else: model = BLOOM(model_name) - local_url = unix_socket_template.format(0) + local_url = unix_socket_template.format(uds_path, 0) server_urls = [local_url] server = aio.server() @@ -142,6 +103,10 @@ def serve(model_name, sharded, shard_directory): server.add_insecure_port(local_url) await server.start() print("Server started at {}".format(local_url)) - await server.wait_for_termination() + try: + await server.wait_for_termination() + except KeyboardInterrupt: + print("Signal received. Shutting down") + await server.stop(0) asyncio.run(serve_inner(model_name, sharded, shard_directory)) diff --git a/server/bloom_inference/utils.py b/server/bloom_inference/utils.py index fe2c913e8..c351806ab 100644 --- a/server/bloom_inference/utils.py +++ b/server/bloom_inference/utils.py @@ -82,7 +82,6 @@ def initialize_torch_distributed(): world_size=world_size, rank=rank, timeout=timedelta(seconds=60), - init_method="tcp://localhost:6000", ) return torch.distributed.distributed_c10d._get_default_group(), rank, world_size diff --git a/server/poetry.lock b/server/poetry.lock index 8100c2008..3feee60dc 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -205,7 +205,7 @@ python-versions = ">=3.7" [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "f3dc5b2420183f2e7e9257e372489409d7bd26d1dcc535fc2558ebca50c988c2" +content-hash = "a4eef5f52e8d046aa883082c865b0865047f611a3240b18250487d4b6e831496" [metadata.files] accelerate = [ diff --git a/server/pyproject.toml b/server/pyproject.toml index 1dd8ae277..3d38f512d 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -11,7 +11,6 @@ bloom-inference-server = 'bloom_inference.cli:app' python = "^3.9" protobuf = "^4.21.7" grpcio = "^1.49.1" -torch = "^1.12.1" typer = "^0.6.1" grpcio-reflection = "^1.49.1" accelerate = "^0.12.0"