From a18e071690cd16ed87051f5622a8d04f794bee30 Mon Sep 17 00:00:00 2001 From: Shirin Yamani Date: Mon, 16 Sep 2024 18:17:37 -0600 Subject: [PATCH] speculative decoding complete guide added --- .../speculative-decoding/gpt2/encoder.py | 133 ++++++++++++++ .../speculative-decoding/gpt2/gpt.py | 142 +++++++++++++++ .../speculative-decoding/gpt2/utils.py | 81 +++++++++ .../conceptual/speculative-decoding/helper.py | 19 ++ .../speculative-decoding/img/image.png | Bin 0 -> 61239 bytes .../conceptual/speculative-decoding/main.py | 143 +++++++++++++++ .../conceptual/speculative-decoding/readme.md | 169 ++++++++++++++++++ .../speculative-decoding/requirements.txt | 6 + 8 files changed, 693 insertions(+) create mode 100644 docs/source/conceptual/speculative-decoding/gpt2/encoder.py create mode 100644 docs/source/conceptual/speculative-decoding/gpt2/gpt.py create mode 100644 docs/source/conceptual/speculative-decoding/gpt2/utils.py create mode 100644 docs/source/conceptual/speculative-decoding/helper.py create mode 100644 docs/source/conceptual/speculative-decoding/img/image.png create mode 100644 docs/source/conceptual/speculative-decoding/main.py create mode 100644 docs/source/conceptual/speculative-decoding/readme.md create mode 100644 docs/source/conceptual/speculative-decoding/requirements.txt diff --git a/docs/source/conceptual/speculative-decoding/gpt2/encoder.py b/docs/source/conceptual/speculative-decoding/gpt2/encoder.py new file mode 100644 index 00000000..15959651 --- /dev/null +++ b/docs/source/conceptual/speculative-decoding/gpt2/encoder.py @@ -0,0 +1,133 @@ +"""Byte pair encoding utilities. + +Copied from: https://github.com/openai/gpt-2/blob/master/src/encoder.py. +""" + +import json +import os +from functools import lru_cache + +import regex as re + + +@lru_cache() +def bytes_to_unicode(): + """ + Returns list of utf-8 byte and a corresponding list of unicode strings. + The reversible bpe codes work on unicode strings. + This means you need a large # of unicode characters in your vocab if you want to avoid UNKs. + When you're at something like a 10B token dataset you end up needing around 5K for decent coverage. + This is a signficant percentage of your normal, say, 32K bpe vocab. + To avoid that, we want lookup tables between utf-8 bytes and unicode strings. + And avoids mapping to whitespace/control characters the bpe code barfs on. + """ + bs = ( + list(range(ord("!"), ord("~") + 1)) + + list(range(ord("ยก"), ord("ยฌ") + 1)) + + list(range(ord("ยฎ"), ord("รฟ") + 1)) + ) + cs = bs[:] + n = 0 + for b in range(2**8): + if b not in bs: + bs.append(b) + cs.append(2**8 + n) + n += 1 + cs = [chr(n) for n in cs] + return dict(zip(bs, cs)) + + +def get_pairs(word): + """Return set of symbol pairs in a word. + Word is represented as tuple of symbols (symbols being variable-length strings). + """ + pairs = set() + prev_char = word[0] + for char in word[1:]: + pairs.add((prev_char, char)) + prev_char = char + return pairs + + +class Encoder: + def __init__(self, encoder, bpe_merges, errors="replace"): + self.encoder = encoder + self.decoder = {v: k for k, v in self.encoder.items()} + self.errors = errors # how to handle errors in decoding + self.byte_encoder = bytes_to_unicode() + self.byte_decoder = {v: k for k, v in self.byte_encoder.items()} + self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges)))) + self.cache = {} + + # Should haved added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions + self.pat = re.compile( + r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""" + ) + + def bpe(self, token): + if token in self.cache: + return self.cache[token] + word = tuple(token) + pairs = get_pairs(word) + + if not pairs: + return token + + while True: + bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf"))) + if bigram not in self.bpe_ranks: + break + first, second = bigram + new_word = [] + i = 0 + while i < len(word): + try: + j = word.index(first, i) + new_word.extend(word[i:j]) + i = j + except Exception: + new_word.extend(word[i:]) + break + + if word[i] == first and i < len(word) - 1 and word[i + 1] == second: + new_word.append(first + second) + i += 2 + else: + new_word.append(word[i]) + i += 1 + new_word = tuple(new_word) + word = new_word + if len(word) == 1: + break + else: + pairs = get_pairs(word) + word = " ".join(word) + self.cache[token] = word + return word + + def encode(self, text): + bpe_tokens = [] + for token in re.findall(self.pat, text): + token = "".join(self.byte_encoder[b] for b in token.encode("utf-8")) + bpe_tokens.extend( + self.encoder[bpe_token] for bpe_token in self.bpe(token).split(" ") + ) + return bpe_tokens + + def decode(self, tokens): + text = "".join([self.decoder[token] for token in tokens]) + text = bytearray([self.byte_decoder[c] for c in text]).decode( + "utf-8", errors=self.errors + ) + return text + + +def get_encoder(model_name, models_dir): + with open(os.path.join(models_dir, model_name, "encoder.json"), "r") as f: + encoder = json.load(f) + with open( + os.path.join(models_dir, model_name, "vocab.bpe"), "r", encoding="utf-8" + ) as f: + bpe_data = f.read() + bpe_merges = [tuple(merge_str.split()) for merge_str in bpe_data.split("\n")[1:-1]] + return Encoder(encoder=encoder, bpe_merges=bpe_merges) diff --git a/docs/source/conceptual/speculative-decoding/gpt2/gpt.py b/docs/source/conceptual/speculative-decoding/gpt2/gpt.py new file mode 100644 index 00000000..fa90b650 --- /dev/null +++ b/docs/source/conceptual/speculative-decoding/gpt2/gpt.py @@ -0,0 +1,142 @@ +import numpy as np + + +def gelu(x): + return 0.5 * x * (1 + np.tanh(np.sqrt(2 / np.pi) * (x + 0.044715 * x**3))) + + +def softmax(x): + exp_x = np.exp(x - np.max(x, axis=-1, keepdims=True)) + return exp_x / np.sum(exp_x, axis=-1, keepdims=True) + + +def layer_norm(x, g, b, eps: float = 1e-5): + mean = np.mean(x, axis=-1, keepdims=True) + variance = np.var(x, axis=-1, keepdims=True) + x = (x - mean) / np.sqrt( + variance + eps + ) # normalize x to have mean=0 and var=1 over last axis + return g * x + b # scale and offset with gamma/beta params + + +def linear(x, w, b): # [m, in], [in, out], [out] -> [m, out] + return x @ w + b + + +def ffn(x, c_fc, c_proj): # [n_seq, n_embd] -> [n_seq, n_embd] + # project up + a = gelu(linear(x, **c_fc)) # [n_seq, n_embd] -> [n_seq, 4*n_embd] + + # project back down + x = linear(a, **c_proj) # [n_seq, 4*n_embd] -> [n_seq, n_embd] + + return x + + +def attention( + q, k, v, mask +): # [n_q, d_k], [n_k, d_k], [n_k, d_v], [n_q, n_k] -> [n_q, d_v] + return softmax(q @ k.T / np.sqrt(q.shape[-1]) + mask) @ v + + +def mha(x, c_attn, c_proj, n_head): # [n_seq, n_embd] -> [n_seq, n_embd] + # qkv projection + x = linear(x, **c_attn) # [n_seq, n_embd] -> [n_seq, 3*n_embd] + + # split into qkv + qkv = np.split(x, 3, axis=-1) # [n_seq, 3*n_embd] -> [3, n_seq, n_embd] + + # split into heads + qkv_heads = list( + map(lambda x: np.split(x, n_head, axis=-1), qkv) + ) # [3, n_seq, n_embd] -> [3, n_head, n_seq, n_embd/n_head] + + # causal mask to hide future inputs from being attended to + causal_mask = (1 - np.tri(x.shape[0], dtype=x.dtype)) * -1e10 # [n_seq, n_seq] + + # perform attention over each head + out_heads = [ + attention(q, k, v, causal_mask) for q, k, v in zip(*qkv_heads) + ] # [3, n_head, n_seq, n_embd/n_head] -> [n_head, n_seq, n_embd/n_head] + + # merge heads + x = np.hstack(out_heads) # [n_head, n_seq, n_embd/n_head] -> [n_seq, n_embd] + + # out projection + x = linear(x, **c_proj) # [n_seq, n_embd] -> [n_seq, n_embd] + + return x + + +def transformer_block( + x, mlp, attn, ln_1, ln_2, n_head +): # [n_seq, n_embd] -> [n_seq, n_embd] + # multi-head causal self attention + x = x + mha( + layer_norm(x, **ln_1), **attn, n_head=n_head + ) # [n_seq, n_embd] -> [n_seq, n_embd] + + # position-wise feed forward network + x = x + ffn(layer_norm(x, **ln_2), **mlp) # [n_seq, n_embd] -> [n_seq, n_embd] + + return x + + +def gpt2(inputs, wte, wpe, blocks, ln_f, n_head): # [n_seq] -> [n_seq, n_vocab] + # token + positional embeddings + x = wte[inputs] + wpe[range(len(inputs))] # [n_seq] -> [n_seq, n_embd] + + # forward pass through n_layer transformer blocks + for block in blocks: + x = transformer_block( + x, **block, n_head=n_head + ) # [n_seq, n_embd] -> [n_seq, n_embd] + + # projection to vocab + x = layer_norm(x, **ln_f) # [n_seq, n_embd] -> [n_seq, n_embd] + return x @ wte.T # [n_seq, n_embd] -> [n_seq, n_vocab] + + +def generate(inputs, params, n_head, n_tokens_to_generate): + from tqdm import tqdm + + for _ in tqdm( + range(n_tokens_to_generate), "generating" + ): # auto-regressive decode loop + logits = gpt2(inputs, **params, n_head=n_head) # model forward pass + next_id = np.argmax(logits[-1]) # greedy sampling + inputs.append(int(next_id)) # append prediction to input + + return inputs[len(inputs) - n_tokens_to_generate :] # only return generated ids + + +def main( + prompt: str, + n_tokens_to_generate: int = 40, + model_size: str = "124M", + models_dir: str = "models", +): + from gpt2.utils import load_encoder_hparams_and_params + + # load encoder, hparams, and params from the released open-ai gpt-2 files + encoder, hparams, params = load_encoder_hparams_and_params(model_size, models_dir) + + # encode the input string using the BPE tokenizer + input_ids = encoder.encode(prompt) + + # make sure we are not surpassing the max sequence length of our model + assert len(input_ids) + n_tokens_to_generate < hparams["n_ctx"] + + # generate output ids + output_ids = generate(input_ids, params, hparams["n_head"], n_tokens_to_generate) + + # decode the ids back into a string + output_text = encoder.decode(output_ids) + + return output_text + + +if __name__ == "__main__": + import fire + + fire.Fire(main) diff --git a/docs/source/conceptual/speculative-decoding/gpt2/utils.py b/docs/source/conceptual/speculative-decoding/gpt2/utils.py new file mode 100644 index 00000000..d82685c2 --- /dev/null +++ b/docs/source/conceptual/speculative-decoding/gpt2/utils.py @@ -0,0 +1,81 @@ +import json +import os +import re + +import numpy as np +import requests +import tensorflow as tf +from tqdm import tqdm + +from gpt2.encoder import get_encoder + + +def download_gpt2_files(model_size, model_dir): + assert model_size in ["124M", "355M", "774M", "1558M"] + for filename in [ + "checkpoint", + "encoder.json", + "hparams.json", + "model.ckpt.data-00000-of-00001", + "model.ckpt.index", + "model.ckpt.meta", + "vocab.bpe", + ]: + url = "https://openaipublic.blob.core.windows.net/gpt-2/models" + r = requests.get(f"{url}/{model_size}/{filename}", stream=True) + r.raise_for_status() + + with open(os.path.join(model_dir, filename), "wb") as f: + file_size = int(r.headers["content-length"]) + chunk_size = 1000 + with tqdm( + ncols=100, + desc="Fetching " + filename, + total=file_size, + unit_scale=True, + ) as pbar: + # 1k for chunk_size, since Ethernet packet size is around 1500 bytes + for chunk in r.iter_content(chunk_size=chunk_size): + f.write(chunk) + pbar.update(chunk_size) + + +def load_gpt2_params_from_tf_ckpt(tf_ckpt_path, hparams): + def set_in_nested_dict(d, keys, val): + if not keys: + return val + if keys[0] not in d: + d[keys[0]] = {} + d[keys[0]] = set_in_nested_dict(d[keys[0]], keys[1:], val) + return d + + params = {"blocks": [{} for _ in range(hparams["n_layer"])]} + for name, _ in tf.train.list_variables(tf_ckpt_path): + array = np.squeeze(tf.train.load_variable(tf_ckpt_path, name)) + name = name[len("model/") :] + if name.startswith("h"): + m = re.match(r"h([0-9]+)/(.*)", name) + n = int(m[1]) + sub_name = m[2] + set_in_nested_dict(params["blocks"][n], sub_name.split("/"), array) + else: + set_in_nested_dict(params, name.split("/"), array) + + return params + + +def load_encoder_hparams_and_params(model_size, models_dir): + assert model_size in ["124M", "355M", "774M", "1558M"] + + model_dir = os.path.join(models_dir, model_size) + tf_ckpt_path = tf.train.latest_checkpoint(model_dir) + if not tf_ckpt_path: # download files if necessary + os.makedirs(model_dir, exist_ok=True) + download_gpt2_files(model_size, model_dir) + tf_ckpt_path = tf.train.latest_checkpoint(model_dir) + + encoder = get_encoder(model_size, models_dir) + hparams = json.load(open(os.path.join(model_dir, "hparams.json"))) + params = load_gpt2_params_from_tf_ckpt(tf_ckpt_path, hparams) + + return encoder, hparams, params diff --git a/docs/source/conceptual/speculative-decoding/helper.py b/docs/source/conceptual/speculative-decoding/helper.py new file mode 100644 index 00000000..6b1b6dff --- /dev/null +++ b/docs/source/conceptual/speculative-decoding/helper.py @@ -0,0 +1,19 @@ +import numpy as np + + +def max_fn(x): + x_max = np.where(x > 0, x, 0) + return x_max / np.sum(x_max) + + +def get_sample(p): + # here p is given bc we wanna allocate the same probability to each token, + # if p=[.25, .25] then uniform else, higher prob will go to higher token + # print(p) + # print(np.arange(p.shape[-1])) + return np.random.choice(np.arange(p.shape[-1]), p=p) + + +# used np.array for the broadcasting feature of the numpy n elementwise operation +# print(max_fn(np.array([1,2,-2,5,5]))) +# print(get_sample(np.array([0.1,0.3, 0.6]))) diff --git a/docs/source/conceptual/speculative-decoding/img/image.png b/docs/source/conceptual/speculative-decoding/img/image.png new file mode 100644 index 0000000000000000000000000000000000000000..87dbd99229b6977e5eaa68a9d0e2d5a095b8aa85 GIT binary patch literal 61239 zcmeEuh)&06^FR0N|66;bBM49wQCI zp5W}|#YF%mL&V##H*bwpC5)w|0rarf$N;z?Q^2ENUx7UcVGr2p>2Lr9*fZR(W9jgJ zKSkI|NBH|1KKa)d7he*h0|3GRiMMYQUEubbQGhQel{T;0E9uP@lq9n0`wQr&>0_(a z;>rTr`^XJh(a!pSM&QlJ1A<7?G6McVK@L{3PKVA{i*-at!{}rOJyM+jt0Qhw!m#!p zuB7mCENvsExb-N*;+g`@0_o}v_Evk@u5cHYmF6$tCYD{M`#hCnt7K^FW->G~iQvA@ zRi`OoeSJNGi;Wzy%Lo4{g1#`?7dY4(u_WK=5gFO(;9qb4^@%Rp`#N*t-(UIXs4ttY zaEB+FOqBfdf6o8?;j;t6{~plSMZ^DUL*^Hq^}PGvckA-;Z|wQ+0epM}M5?*oB`Ugi z|FwX8x#a&kfY{>$hjPjGTH?dUKgR#H3})(o4}AImrXnD6g}sT`O&ZK8Wk| z9UJk7kHG9b3O8a+(QpSVl9H>)>` zdH1jyZ80222cjl-8^`vtsp|Z0PQjjr2B5$M&v@|lU95_=+h99Wd4YOczqRK@{ipkm z@&Hw4JZ+bJiR#iK5x(W(S4MV|=CTVYLO3PEdwBJLbpi-wqz-@a)4aMzaM?Kqlx zSa%(ANXS?ro9tdtP~d*ks1b&Bor`&Qbw|R|XbLGjbZM8p@NuO5#~(DGBa-s89L-(| zT&hH-h?Y)L+ZgLja~_yPY6`R6--ydL-4tAVJB?11E;i)NV+m@_$Fmfifhv?10=ir}kFEq0J$D>S&4Oz3hgWM~B;=spH-^t818SY;O6QtKQM zvw7z`8ARr}OEP(rwdU@=@NH7FHdv7ZS1EY80)7_*6LJV7ne3wd3LtH zf6LDjRd0Wv<8Ot8q_qEqm37~IP7S|P9yMr=b+SHT zWk!>#wh_1dUcU+-tCYq;-IpeKouI^RZPC}<&L?=)?(&@>zP|?&5`=v&6LR-mh%4N%E(iK3CdhDkrd3j zZ&yJyUZ*w9rL7x3N+Z#g{ueR>djal2xI@(QR9`D9B4T{5Yn+;Uw&l}|De zoZwXp`1Yj2-l+~4c-}qLFNxtXUNH@eEN(u3*yWnrW8aT{{*|Lfdw5TBMV{KJHXoU3-C6H&6 z<4YiE)u&U6slK|t|H*&65#gCMv7&6*``MT7*}*2BtdcB>PVKADI;LmsqCz@QiWyJ7 zmWG+fYX5U~MYLqKyz4KU=#64VI8Pvfo9-zEDMsUssgKpNjZPT9vp=2morVEi1)&jS z;#*SZ-FK+>7Z%y#oRIYihSu5%=I{Too-!vyv({PUT1xiSyWKS2b68wgaW}VxE?ieT z>yV12{j#Z=#*g4q-97SKWJt?y3)vQ%GN#T0+^JSystz%wu7#?AIBu_m8g2H1vD21N z70mHgR`u>9;x}5T^TIT*jgEO8UhgxD+sm?wNB27gbhpsy@@kWE%xMag z+cX6nf={G#^t{-XY%%)n26!FwP(!^pk>DlE1Agfy66g{@G<00IUvaQ1d8RondzBnu z7k=l0x0*R6u~c3PQ=b_S|HldN=@C4r&lf$T6`^uGq)eNm%yW_am)9e_6!X(%kM760 zZw7w3s#Rsg{}$)mAbl}Nd6LDi;r*i5;09)sQzUNxm@P+n4W0UGi|44)_-Nne+rplY zW#rR3CryX$>%M6d+|1z}+j1Cx5P}sQO|!w2pZb-K*Z2+QLdE{>-9H^Ubl-@a zRD1b-sCBhRXFWB8U*F9NZ9wT?k#4uMtz@urJeOI})y&p26MG%f;(i&Fw)ZpSWy>?A z51#W^WXHk^RCAaIZbtk4uN}^;C)Dbq5ixC9gnFB~*a-fi#J-NkKEeA1HurnqP$P+T zRLRbzV$M;jhPvoApCR@bFsVe9RFE1&Y7!46*7+i@c(bS|m_wrD7AYJ*+9}BWjsC0M zZIY#V_fLK}3%VFL@ru;MgCSvBLmymN5nx78tn|z~kN-HnW4X~R_GApJhMm23Cb}QJ z>8>&B7w|WvvbLkmmA7iwwQeqRNHs*N^tyJrb1ug+L$5o^*>)YPLD7Z@u+S5-c^_F# zb2vQG>ILHtN*bIwJvg2=7pUU*6*JP(Ybre79=IQMDjey!e4(?N(5WPu&T;gXnA~of znyS}jMvb??)wyg+fxznO$yLhCTJOwjSX>=v_el7-j{-fyTm1Pe1ZZ%s^vs+KZQ{z@ zjob>2QQOC%l&j{2%vDw!t9`_w4t?+mUlfBS98c29EAGfNchEe;@Il@Mx98Gy=oOla z;JCWGnb&S5WPGNuM`)Y!*!4yRhY`ff3X9_;B0Bb~r58z(zbN6j{N9R*11Z4!6lYP1 zPtL_30~4HI93n; zzS$74RI;<|s>Z+m%8*uF?52ATy1%>ZExaHFUEX((%s6t^?k!CDjV(8@$q6w}GTr<{ z(*JR5e%V~EquF)9;=R_>u5c*z@qVMXwSH4C;+D9?FNKt3?p(Uscfs{=7l{#0-xTUy z2PJ|!W-V4^&Iq&rgBeta3l6(AUh{7nTMqfFSdv$(%pobDQ3Y(_P~i}-jBL!?>RZSAhqma~HUIHRBA zkANRcP=*C`+%|o+oKLc4lDIXW<|fsq%i(1-q5Glg?her@LbpRX;0(AYS)_v&;jcq`{I76V;blT@=lt ziHi*;lsY?Amr)kf@ui6c`yv)YNPxR;NtWOs`XUa5QQEl-;A0MP8E-G2N8%?96($&7 zDYE_a9mng^X>LC+&8ztqBGTMXMrH3!w3!4dKi&OeV=q-%|GF4P zJa0F(?Z$<)P(90;+~RRwH)R^7pgL^wS^qreG;>aDJ?L(oJsd6BoudrEen4p(TLRq% z!u=Mf@z9^%lg^r6AL9mI`&!of&yIpL`9MKeQ&v!F5ez@yZJdLljm!Cy2pCuN zE`4|SVpxA%YJHT-i$=j*5O^rykSKHGx^W^^?kmq8I@k`8D)iro@S{R zs9mndR0**3H23xrH)Hbh&}F(he*T&X>E391wwsCL@Gd9*7AG>4plq!PP-TtKJ z&+~Em|Ir9I<)kAzhz@itzp>RJHMgsZ;!(!^N#Vx*w6w;xST5I5)~ge6S!KX(6F8s4a~?g=`nnjnZ^F(jqBJdUdu^)XReUT$kWOnXOm-QZ2^GH#}UNXpOPSJl234;kbbbQM@CI4_7m?3R0 z?BvUWshiSj|I035_mCn$cy^%R)G@C0AM1_{Vb<|Ok}Cc)6d*fnD7kz7g^mBQZnYm~ zol7dW5z~K$VgwtC>@A2ODAme zhY1i)z-B3NK3+#~|54$+sPR&tO3O(fW#kE7YOzMmoWp3i2_$*issE*D1l{GIMK2>o z+gotF_C-KyIUcfZ+AnNPbaB7fTb3=_#ZqMx3xtcaCKmJnItX3|%X(~ylDVxdi$@UO z4)hs1V?C3&MksULRlX8%zPtJH<76mh_TxULz}Yb5ZV#gEZ$-D}zw5jBteBkcO}8*M!rTi%=w9Uw|9v~z4V)iSSmR4FUukKjhb-Vb z2#^*(+vt24dD|5ky#ms{U{BK*SGREyw9qD<$DplrKm!jA4M8B*b9GqpNnF-m+m^M> zS8*DG$H%fZE~Dt&g6Crj3Rn3xSsen&1?dZXcfpgC;t?L5+#Y_4?hR~=vX2qKjaO+9 zd<(&y5Z>57R0-zy!g=!-74h?SAf_y#-X^Ks%DS?p$O89+`qB5!(hZTD!CKN2$nSeP zYb-Bn&m@`a{g2D^R(mw8s>|ESc}`j`i+i?#wGwo5*J@9QxH~05lGO7FPmLmN!D{k9 zDF*?5gMq}>dPLirvB`h751OIwWQkU7>&gYwbwh7=cQ-bTEUA!JvKT5JZWb+K6t{2e zjqf~}QwOJ4Zv+XlBpDG}ROF)Pk%K^>Nj_od zc!h||ZgPY7>G%7)5HY|K6>#(Z=-WPb+QF6IonG<=YZ;H`aHY7aZplYxPP@bZHKJcKs8}zib`ypK(E*|0; zc~z(J*l(-_N8>STmRWrwv;oT)e|8{1@VX#CEJ~QJu}q3YM??{Q`NqfA{+oPVql>zx zy84=qY6Kz?*e{M}+F`tdv4dnf@&0NWMYw~7#kyf-jPl;fnG)dg419xwnshM}y-~S{ zK_SVaFz0oekQ?XBr5}fDGA}VL>#*YW!dN){X_ND$;7~lZ8(VHA}A`5unn+`eCU5C#$u$dp#2N`xG1_&|+MEY+Q%(H&E zC@Z19zcHh1o0*C{Bd6JCZAaikO^Giy0%GLqYFHhHq5zP1d(f35DiOkE&@^f}xhx(n zIcBj^5XjQwXrUpJIXa>ZQ2&Yat6gpF=9n)Awd&G@ybt!KIkYbX$T2 zGzeCUUcs_eY5cAy-^5Abo>k9ZoyjLmu=?OGR+)~%LR^jYq70`jj)~VTNayaPBeOVC z*MOCks!fh-`A!7ovJFJ&obcJsQtLcm;8j8_lUzoN;?QOu5<-!5bo2$jq>o16-bmT> zApG4b?tmXF&ue|<|EO2p(A2*F zNab~o*=i4Zo>L1AQRkxf+M)O6{WVFdPVYOLy@q`MLNps==fj2D!23IN=av&0Gj8hz z@b*U5>Gc?zCLx+?L*R$;d4KgELv=|{YE9hB&R!OI`BtSub)m}xIX;%3=T+CC&nvRo z?<~F|0+sycvkAiMNW33Zzu;h{zg{`~+oNxmIBeWVLBj7Jp>oSYWx_5Hu6*brrO z)Mh7o1Sc>VoqyLNR{Vht!o;nu+mr+Q{X51bA@9R9cew8?NyoQ9Cr94Vje&iUZ}3Dk zF{3{-lXM;tspxoCryW=sy9K><-d`ijetz&Kf+EeLx!j!nYW3Aw*%u0_;!WeTxP0d% zIwu=S@;XbKA79GLC^%8$7vByK*j2u@Fz=co5npJnAah+xXinzo^AHZO z^V2KWlPBNf(Nw(JIXV6KQW5tL%fV+rydl)Mt-b?uTofl@3f-`}!)A5h3F*oHdihRT z6leQ7X81BfevzlX-R*FOsFxSkfs7n6K#(*ixjLqU7Z5nRpwnw+lzl%p!U<^R4hu)b2{M&~)MeK&pnsRXtY%l7`%f!VWH)v%d zT%-wm4SNLhK+~6{=Vp~rSsQt@_^6Bg%Zb0vxqzZj?nE6pUg;9+N;lke5g4A1Ze<0% zupZ6PHbig}kbp&n+mBb|H2BMdaL?I{EAepv=!Aq{QTv6uPTvK;sAjm8zVcZx^Y)(5^DRH_^q8+LHV_tN4HkuFyJ%MQAIK!FyE zF*6Q=Dp|p?5+$32I;0fQbK`=Zl5}(71KRU4swnw_bD-p}aB0?%!%l48IEFv_v+fhx zl(pk5zpwmprcqYgZa=_8F!@X$@LrJ`=fDZ?d8|9zc1!A}YI#l+T>L)QDyi6#qFuVo zZ)Q*J52s?fe(UcbHUGXHfM8J2EB0a8N-9uXpdoDKY>m6WJHf5r`^wIQYxFFe+uUiT zbLO4;i?=TU;)!omZ3G^wmk-Dx6Nw%#u^xA0}U%A`NJEcSLyhIN&u zGX_p^4Ik5xlwMx{-j=1^iodB#omN^Wk?}SKT-gfZE3eyuiC~egB*_gqSjakI zeoB01rpm$V#SP!rz4|>wt4B+;kB#dq{XM#{6CpR(#?&DHK}2y%Y5;uQTJBm`0b$DX zWhSe+M53Cbxj)-JPy(-M`f12n{$I+|=cDrs1ZCUvmnX$VKntTIBZAR@U24U`)*acz zV0G<;*N6s*Wk6r_?smY;D_&E-gykd!re33#auCVPw``Mp5A&`yfVs*=8K+@3Op53aFj z{-R;&*FVQ7XUMko<3uA&OD5c>-Nbi`|p5)d|n!VnAHvk!s}|$-_&7SSSI`A&p+9G2_ZKK4EKo}nH-eG z>TCE0WF|JIEQ^zVP@Huc@t1bm;toN07`3r{?M+eAOr}Xs5jATZ9CMrjU8|-7`w{_- zsJTZxZ#0hgiMZvmi?3dP3O*J&U}55fw~0gUZ2oMr5jrQZY-B~n=xkR3PGaRptk`;g z7R^_#nI8c}M#jdow$c_b5IFJw#?E z_&A98IPmG@B*zgMR1*X+6#%$d@UnWT=<8s?M$_wH0iBycNf~cr;mq8c#ba=wO7i8L zzH(fW&{72dY5e0C;nEb$JYc+h{n~;f`e&#T&l;nqV5zNu?A=|7fj1X<`A6zvX3L>f zvmUXO!tyGZ5A;waCYBGjIT2`fqO1&%@(*bgRf&f_4;1sMg++7%zf7)=DdzT&AgT9* z`x}`k*l||xS9kY%VRqi~?NC$Pk`{LZdf;Evt~QNgbn_gDocE|g zIo9&=2-Mxm;YUtVKI>x~2d<4|`v^-%2R8qkZdHS)%3uC^Lr);w{Y&k=didiLCft(+ z|A8jl)w!CXmEa~jtwj?Sehn$;xLC`PX0hv3Pklg6msnuck6^~~9|C~3*F~h7X59JR zF&zS{KR6ZNsN&T+d*5fjla@BE#}fWd1%%w3MI>Hn<*efZOAreMwlZmXC9s;M@g}gb z`D)`tNVIIn#+6E50%!hcIc*KUIL)cR;jLGx#`wv}ihg}+=o=u&HVM_!f_O$BkVaQ^ zsoQ^<(UVBPIOgiRp@!3wX*gKPWWE$TpRo2d)p>6C%bi<~a_FDXF{E?G>|Zrh1WgvD*%LJ<>*xX~6L=;xMy3~|#e>WFD} zqHj^z7q%1=AzXj0T;?to8rT)9f&aZ@0{dZTU6;FLqKf^$Nr%l717M9! zoNM>C>k=pBM#V*LT&r=)e`u5i^ z8OuLXg^#^>>Dn%F=2=gq@KJ_g?v^W0ik(V^u2O-`Lo6?vYscb7qCRq1;v@a9SmBGq zvDDgq+T1Ko@R3qD-+3!!6)mHE%kal&{Ws|_{vFj_dk>zuJ$28?UTi+d0>@<=i>3LaVP=$#o-|(j?NAm~eXm23Xq3w;+1u`$ z@2#!%Se4fU>tKoQ$i>>#+_p1JLsP{XoVSOE^&`L2iRX&L(+A0uZ&d`>IwGBK&~+#j z9~cCzk?m&Ek^8G4M%; z#eu(ZLfY0NY_AW%7GDwY!u*HK%fI~PgJM=TC{PpMv6=X9;`$q1u{(w3mm@aT1J2%w zA-+VBe(8IVHSHFd$vf4gg<1? zGLu=mEAl%p4rL~@Mr(Q_jwz?Owz+^MKr_N zKMR!;H&y3^tgvt41Uw^<~WG4KFa#%Nd8$3GX>>wSoZUy@J6 z75Zw;qXX6SlWd4@n>(k!^Sf9URhe_31|qAV*D%SAVUz%#&^nA(fHIuc~n0Px;*CQ6oU~rd{8U zy~U#lF#xAW?d#r{oj&m>GHaRMHJKMxCtz@9LuN*o_X0<TLJXVw z_4cvC>+)wB(b$oy^V~2|L=~PXvy_PX(FX~YCGLA1C0^(Dl9WGkSr`REHi#mA^Morf z<>P?4VH@*tHdEi#4`j9PmuwcP4m~NsYS=~>_QvkBbMh|`t4UHz$&72HRqS|OY)sxJ z5tE^@mP)WnUWE6(Ne!wJjq0W`VRD2P1p4A5xro(ZCdP0#VjnRlOd`^}b3ll%4Gi4p z=YIX08DxyaqVt@f<;U^-ZNcdYhVddoJFbLbXN2ui=C>A6D50#m3Jp|H+X*WDo7Hr! zh}X#Duh-_kH3J%tk_LU*9B?+`kZr8vJZp7AjqVZn#qDDV!ACpzQ=Qte{AQIvDG*O|>b+a;;Uzw^zn>P|?BX{b3bu}e&7cc7SF%JWWNb{V_BH6xHwHTDZ3aK+X?adoZBbj;7(XpPceuZzRNxdowP@A zJD3*1#d99gOBhq|uE&B+X{`in<*8)KW6(DW1D?%k8*b2X{r4K`BCJYR*^R8;NWOB# z$f37jAK^H0qyOqRr7-Y-7_xo>**3Lsy&$!Hj<13@vbAVA)xQ(}MsJVPC8<11KU0Sa zGvTN$%j6;&<8cc#GyI_B$hJ3f;}5*^J%{ehIF7}|L7;B&A}9P#5_&r?Z^D^_l_54I z#LjBnP8RRxsW9~+QAYJXO&d_Y4vK{6b|4jB$+?8x7Tds@iPlBYD|zgsh);ii}hEr0vH=#w}HcfG+<2dc!UhTdSj(f^e zaYIU?ZSjNfB#eizgo&3c>8r?xWo^me$j%x1|25aZvB?>?ZR)HrqM@p(weQSY7^(IB zSoxuE!dBV}Jfv!>V||vOsD?k?<@WwG*Y|P_A`!Vcuflm5_LcOei!A5)0oQ!Oe0I+) zH}YNsO~a{BTG}EG>Fj5LIb^Yr%^JrA=y?($+)}iz_KgtBo~P0k0Rv5_Ge$h0&dr8~0!s_*>93QXh zjv9eXB4)}$$q=hQ5*}$7@B4GY%jWtt)q%otN^~*vu*qE5m*NfXwXC2gjE%9k&Ylys z0Me92==W#|NIltfS6^jccqM_%Cmz>>oDPsBN6k_{k>b~N%2^t@`;;IE060%-2OK(thG2A?t5s~(b)(-jH$c28o)we(HobgMt zJn>f;T|eEwSi5EQdEra{=a(UEb!>9D=$g{E&vsL2#tO;Lf>-XsqslqF8-|96897eYWmC`}@ z&`b7+bu!raHu#ErO760&3|KF5W&@SWE7Wb$Gnjcz**0HD?pk>%GEWNIPZ_q-;2co5 z@a%F$xEOUgDo6@Y`jH?D;SoQk3Rlx zbsPrI1vhi^lnYmDD0C{PCMExtYiza#dS)w9-U>xlTm0>WjqV2E(PSOW89IJw#zOGJ ze@{0jjhNKIU=5l>kB=0ce#M&<>VEOYVtU^IMst^pi}A<2<3!(=)s2=xbtHpfh>U?e zeYMYG`VC)Wf=}}mZ@)XqOsrElIuV>@Ls=>?;|GpvX++zz4El#Wbc7XsxH-*uvUZh^l#UzD!t_sU{U(b9DTu6c`|`Cx zO0m0Yl768=N3%hRs|hbr4Pg=1*C&;FpdS+g%Q6vJ1_N7yb~*=)f+kv@x)_-r(Fmib zLiaVb11%3)s};>}kcNAB0VL+Qm>!mCukR(E?*>PKdN1Y-%+<1d33dive; zC&VN?&B?6yJReQ?8UE?|@U_)#!>f1>JyFyh!*vx0dhDlob2iEUE~>;9AtH?;6WYj}Ybj&CJt_vq}{BwiU~9aR+LxRvIFcu|JGY%M+`*u}1oq42UN z!sffmTHI129UC!XWX)&t=M5ZB#J?p|L>X`X;8T=oi&|+lX%|5uSCXr5_m5Y2aw%*q zxSCq2>HLg3S;5h|nLxMf8swFn`*YK^IYEX0cgw;%W8v0QmJ^is_=2=zDZ_|juvO^W zSidx9+lyJme@ChaYY!IO@py#6`8D}ZcTSre0W#*NA(K)|cGfgYPT!9)1{hkWeoh2S zLK<>p-HI{`WAvTX9metTm60y&;kzo|ZaSS0(t%sm28(m9^_r>;?EORz&F+E}cBv*R zo^l^whs~pzx?2I7b!9|0)^iScphUs|V5x=qd+-uQ%T^W+&u0CoX`ry!7o5J4G`D9G zA;a6Wuyzrr71wU84K$~;0Ba}v}>s04J z-YM2pcP7SzmRvzVyB!vwfLB;Z#0--eo}GmYHD1l>FAh*BTL^j{E_LxzO3gfVCD53p z3*ZytZPI*WlsM-?cl-93r%%#Y#+#0I@Do?wyvV-$_>bJXb300a1FSwjln}sC;wl=q z?8LD!uH{vD>j5h$;M&n-VDHtHhReL}lJ!N&x(c`Mav}dZ{-Cmn%sF;GSA5C|;gR=+ ziEPXLI)dm-ZFBaBjN`#y9Dp~quMQk)i}#le>MQG71Gwm@dVaF+Oc;uQg6viDRSiwy=?Tqiza%O zOKD-Q;FpC8ECSVNOL@<-W}}`sUzNx&7@zWob>08Y*c>f^C_f6=M3z3aAbI3qnTH9U6=Z_>bC5zDJ9p zdCv~41n8D0Uv{8L1DSo6PkXxEE2HHNHDhXU-mwJq1VAlD1J1k9;!B2iTMXk|l>Uk+ z@xCkY6JIGiZm*74(aDu$NfAapvt{Y$EJ>%7T_E70uHGc(*(a6y8rrFje%`4`PoRZC zvgF^pufMBc`%s&*g5PqilFGI&(%O7VTdr{=j&sG%G4P%&H-$0_)klN(fbG8EcShy} zo@JZVfBIY)VSO$c;e33i-@>1yus`qy7`}Nsuw1Nnv#+0QUA`rt*7J!bg}AHItQt@T zAKE%Z!Gdn$UVsy(0)&9%)?cw??wh)>CKujGd{4L!r@qw2^!)ne<=5x0bQ)O7gIjFnAFp}i5Zloz$;~Ys&@|*efM8Ku_1VfK@1$N)o^E=(j zcN7|sw{~fm4^R{4LehH8+CHq8FywFwQmkqYVP(TLt_IEd~FvTMMt*uB^1x(lD^CDJ2RCSvFRhf<(LHaAyi)QYE zr|oHEQxj8C=IaeM!O335DnU^XxykRQHrB6-b>Cg$<5@K8%KB=`ptDD9(kFqUR zID`B;a$D_EJH@~eo~H@XOBqMe?wzu}+hnwn z^XDy}w{HJV&*vxnarW9oT4-EWUpF>3_Tm_B;Icx+Ngnpi6{~J*|7okFyQ_ztN{*+d*Aj)#Qu9ez{#8HnCpBBB|v=xgi^fa65cgu?@yqHF%5vp~#ydQDhphWn| zdH?w^dHQmm1QPC1acHwXD}M|9R<_21`qnDxv$q|x&f!k}PnA6N4T@TRI8bzP;#7cv zWZF&aP-iRmuNv0i53UV_eSvj2t;DT@t`3rULoyQDO8$|ve0$1vpUC(%ZhN94WHk$QwC8|0;Is*gicvb76&aOX94UUV za#R4)Kjo+sEHyGxfTh_&I0kSRe87LK)atN|P>Mf%C~1AQ(7l;DV2{fEtYp5D?@8@f z?Y?l~IHC{4;i)v(sBywlQ86OyUCaR0_rTl0Raj4bhw!rCK-TzCeFw$IRXwmYtO`#= zzu*CB&qhVdDeK~C^b>KKcD-5&lDWN(^vK5&j-diJdKny!vn(buSw_~{UrW!McpXkH z-k*FE0~<{aC1G<}Z+Y3#0FR^f27JC?0^k#Z1}6T6>v;}JSJ4l(*`7!^DlQ3Vs=4`g z$c$3S{?ttz2p5u;S4>!Adm(bkaaM55%&$^i>}O=PJpB`sWTRX>vFhL*F&F;cHT7~N zjqBa$MXF*QW$Ud>-(B9vJ2eLH3b%N`!+!y&rnDN4=J$4xIjjgo4?;8=lUmFvJDRb4 zN~wUw@er2eDZ>P{o3Fm{1ehEtg*sjmx%OFDohqlsG6P;V_{qAzeFm0wYrHwTOC?JA zm9R*?dE6Y%LzB+XdK?0i5pw>@2)Jl<^r16FMbrc9KU*F>Y6MHToQk$?T~+Z}=J)Sq z=q{{O^U;bH8lCV3(tQ|-aStZ_z?e6$t);_kfPD|ED}VrLWgJUJM)~zQ^=~Wc%RCDY8SDljUx>JSBm_x z4LVu)KK`$~(BlW!_&HGWYxXXl!aWY8)XgDFvcE-Z&04U$>5@EvrKNp4(DgE_NNfh2sqAhp`lm`s^2j~cLskc zL>CX9Ik1U>r(;i-s@?3C*EF`~TB&p33o5rG0zCbgarTYn`Xo^(sk%n;>wX-jj0uH4 zegCz2{brODyJ_{dq8EO>%ImAC19nP`e&trz{J6pYT9Dr0|bGwkj3ZFQ>Ca8J#H{SToSB2qI|&Mn6qVC)%=S z5T7|Pz~qVsfr?pdTD~$OuUe^ww%1`C&WD?77jI z9nUB}*@2T4A<*bCvGzKK1|EiC>D!2or<2q=8*R;cyY}>b-#(4-OB17kQ?L9rm_CJpWTI5d_4%xiQL3)+6yaSC%Y61}u9HcButW+U;abVP zqapiXj3a$?9wZ6tY0B(Z{u02HceL=yi?~WjO*a;VX!Cw*VA2 zHFydmjV>(eUXzYzH%CGAl37>^YYEVDG!B~Q;mcw)x@NVoZ3LK$c-|AOcSxC?WFMtP zq?J)xEmbn)a@c%XQj($m`~K8PzL8#M-$i$HOEC8AutcWTq>jzP68n+WHw(4Dm-WpA z8nmLh`fqW(*r)Hty%_oQq+4f!Tv62uR%amlMw${QU~Y}pc3;%6KzWW)&dHIQn=}QD zBWq}Fxmslvp6)mRkU zNpuCdH)HZJF&0(j&LovS*t{%#rk=g|s+&dI50mqE5-&xKAXsV(-LK6()UF*~ga4Ml zb@pw|7L4ELU~q{{P$^i%GY@lISz^M35;Z@I;+)GG4NRqQBS}ht%`-6Hfm;!AZOt5_ z!Ypero5UHH%mj9{e}Xp6^;d(dYy7>(MK@DQ(1F?ffM0t^R~;We%8BKkFtVb9htp_8 zT&-b_U1yCUUe)i^F@_>ZWM@|nPkG4s9hiiW z58pX_a6Q>*SEjj?22g41 z)_cWf3JEH=eWgn)&780^-L037Zx`$pmp#KW6s)KS69{_3ef2(FR@gXeu~`+-E@rSj zEjf91(0_%lNF6299~JdIU%xF^aAM3T|AKQQw+ANrz<#yUIebuZPa1 zh)L1B*z^+~AJJMv)Zt@M9PDTwau5WQWJY1U6$!∨7Fig%$9+E6@PRwb|$$MrC#& zX}7UdB@@H258mmrcAq)4x$)jYC@Sk5^nb{D=cr1?Swawr$tWwrx+E zY}=kNxu%-Vo_XK*obOv_t^H@M)xNu*A1*z zhOJKCbfT||V)^|K2UAf{+j`rjKpJ}29Cp#Ch1s=i7Y=fC!v1bote!DCG) z%t=u6e)k04kP)qfpJ2yH-8-5Xz4`x@=&F^Gk?I2ljAu{#tC9zB$UtkFbjIFaTq|hi zS4;P&Ju}TrsFjhj)g3O(>dV^n4k;>qSs4&MYA3ZjJW(ho&Ii|C(aM2Ba`1c<0QP*5 zO^OUg{?l2ZUR(#W>C;TZt>@NsS3ck$G6_%Uq56cL+mRpne`4%^Z3BOG!mOw2t{Sj< zm6BC{rLnO8B%F>=;pN+t2*sPaavlI5uH#Ge@p-uhvo9c9CT*v47WLxzvDz(r69e-~ zngg32nRrN3UKuFN7LoI0_q%}doJ<6QY|o(HUg@<2MAdZyb=GH2&n;9kyHdil&1Tpt zY38kF(xm{Bb?y>X{D8DzYi6WME3p0mJoz!w*tyF>mGvu$*U?nb#{TWh_JOqAXCR`7s~e?a-8)m%^Gxtg-F_oHfw6AFF6P|Ez9K7b_LhsE~cQV zaqn2DE%->TmXyAK4gbk7-OK9Uu%Vl-$&eI?!56s0ajmab=S{3;2ZG(X)#NS>c7fIb zZ=1V99ktf~k+JvV?s=R)U2QU-sJUJ_qd4FG?65);KEgv~nwUZ`9>j@n+PHjo8bH#} z0-uwz@&W0Eat zO3{z&G5%rV@!b$a<*%@QjTokh!1MVHL3PYuq2tMZ7$PtXo_pfD+!5`r!AvmjFOBuL zGx!1}Z4@Rlh#_@!L5vKlSB%nV5`X254Pa0!726+oKW{I}L7B2BJ=>Po|YLh*9*JY27@PX4N|2$W;EduNN&vEd54K5x;6TCW`0xnRo zY$$3w)--U~wP|sJwy$(?(TS%d^8HW0wJQPMO_vsxtIck;DMsWZd-#8;Q5e^G{#{$g zT2p}aH(1@lf1uF@TLaaV!5@wIBA6mFcs-`}X~_;&ivkOT3`@QJAO2IYXEMUT5vkoa zau1~_!M?rAaTR~qDU>W$uDlR+ZbSxsghA4$5^G9K9h?G1Mi|s?mqUudn z?p%laWG;4IP--qWSbqWrmD0!^P@3`bg9ysN3S-$>hfTIMeN6cuW|No^#A)@c%4Ei= z!jw)@T10-S+8-7)t8siE?B1Dx9r=!??|zz^84O!nvw~`b+;6>&MxMpV{9j^H=-OM?ZAd*rKN3p0q7eQS%3*|(h3lxrWPVnbnZFi4 zQA%_6>vesbu&n#j8N2P>M@orx#C`eG0bzddp1E2*YTH9YydcOlJ^uvHztxEn$t7O9mX8s>f zW+;t7F?aohFO-dvlWCWQ+ww}vbY$c{HsRFio*{>ny1&n2w6KlrH%nQmwWnCUS0Yd- zZ3p^`D=#*J;q5{bNqUme!QN);p6J!b6E1huU19l(p2HlP^dvMz_JgkAnx5(;LekA) zJm_T83YT`_rEV?LbEo^O#e=(vPCW2@`48QH`$aAxr?hW-Te7qQ2`cJ=p2NCPC<`$= zeEDL426(cB!=b4lYqhzFp*U1_@5WSzTr0AvYU+Q_uSUH)oztx|hljBfUkNIBA@-T# za6O{cELXL|r8!N>*ZjcGuCuZg$MQ#@)zb>#8VKds9;=i5ry@uX_ecBR-n}%E*?tfh zs_g1I`*MX5?66t9G6oW!m}NK4P~&f-wrWND@%rY3dsq%1XD}!`NR=K2x@K@sWQB6O z_e{~Rvg7deb=!-u>tHih5KvWDm$-exEij|^qBo|F1lo4Rs;F=q`(;Q zqff8heW8o36&#H=AyM{Y`0taOf5V{Ar6k~Uv3JH|xOin}zUA=!!#&x2$In$&lj`4h zg*6Yf=z3$`NqtOu#o^j~Yf7BsLrf5b|2*e)H;`$o*vC3=lky_2rk2bWJ0*CI$UmlHr4K@4?5?`NOQzS zD}C-XCiFc@lbf%?pH9POe>l^=!v1p(Jb=_hao&lX6gIWS3C+0x?OlJVS1kQ~Y@faQ zm;JgqZJd&Acqa9K$_E5MkVe&$s_YZ|4wFmU@r{4UhdN8YOai!}^$+h?c_2+G%4sHS za7cUYoG!F6Iix=mb}94a^tmoFYNP%61paCgbyCJOSa!si{bt;R9U!Mxz_Cs zPR7hIs&5|#dZY)(UemAR18XeGL?HW2BVVMga$*%URVlL6JD?SSPG!XR^SjOIK<<+( z3pSe^w<=T+``Qf0?mb>tHSMk>%V7wRX?p*!2POQzPD@PQlr5!zQ%uV$nHw^ryB0d$ zdY1j*VrAp^HBa4hrN$N?PerkcSl05v_#yK{-|-#BCD32Sd( z1P(dewK}p(L8>~8BCWLFeD9stlUE;no_+Q1FgjD`u$KOumL+n1vHum!iI90p;^X14 zz6%1@-?eVf-VyR8xy3y4gWO2Jqo9z7khETmIF3$v@3sCW4%Y z9*M)PgS0u5Q+@h+;5@yD?vHl&p;41sPuJN!w7UIc?!(2?6*~F-{#2KTPxw<=J^N3M zG5y8+v$+|noQlC^n!Y|YzzAmy^nX<67U^6@Us&$n(pYw=>`b@}fvsiF7@ z%oeIL*(rS0U;5xOJb_E?PPha`*=+6a-nv`Ol2D zxqz+|UTsai4_#L5WOD~sV}+k>e}@O49)1MSSX1*o>U^*;cOKJW;hMfo9`PSO;fJGD zY|ULSgsQgl9HqfY{5I%wlJ~ih8K2~S18YoN`NBkBH2RHS3;7Yfh z#QGn7wYmG}i4GQ4-Dmk+mO(D!Gk#T8r-s=5{PuIu=Hs9>o@1h<^?DcsbP@jf{VNUJ zEfJSK4z`>u7}1a8#ri;eP%9RJ?xX^U=10b_zd{)SJ+WDK{XG5Xbg$Z(0HWXnqscRy zOR#-B@X|qqxVruz)OOi@MY%feOPkhi-j)B_XTfo$OE+E@YkrK^jdL*{sMAcKQs^F;`e=Gj`wqs5%9%U|)fdz@wHo68>M^*pAbx ziC|%MgdNNB*Gh)}+Vj+pTwjeE!l=<%3%@@pZ1`OfO#DR~7l-=iG<;qVg#~@9$~h4p zl2SOwXjxc$bL{WKP|EU#o%p^US7L&9IxCUxMEDb0FOvuTJFDWEK!4r%-?*y&=%XSN zxE^VAp{JNVogAW3kbO3ed{p|LpJO!SPtU^|?$l z+Wxy?x(4H1R?nvD6G^~-4d6#$fZKM%+wfG1gLc6s+BT-UyZh6E9CI%))suXae2wH|8PJB9GVOnjlaio6GOoCI79e{x@AB(6^n=+7u+J02o{7;Aa@r`7&*ccdgFwKa z_ayQ!W51dvneI0>?YpmlZs3>QRvYXYV*eMT;2qaR1rD#X!iYBe?Jk$o8MSgnLLuLm z4V(Jy9xwLS%iZt!8}+`&^e?1AO8$3%n3x!7Bmz$^+hIY^Uwalp_nFo8eRwzud7cNb z96$2}eNIkKi38r97D%AE&^DXQCT|Z%tvv3hxdSfL^1+mJlS4r*gWwp5KVL>=5FC^Q zAD6}f)Z>7o^SAr_PhXr6xGm#>*+nJck0R{evxoD=Z@(APrL*G=OG0=C6q8wI-9JB` zgA;F(T!& z5+OM@Zk8nY5yhMiKgE#>+;n}L7G2dnmr<>h{Ezlyn6?Hp4M(waBzu=(`fbQ!&Cr(G zde_Of1z~l6p{oHk74?(;!OnV_0Gzt=IVx}7hl?)5T2a^d;4IU-NNLX5sWko*ivI%+*(jQ_$MY6~M#80}~uXYKwkUO6CPiyn%^- zS%46YLZyvGmGL`?6+A?enINr7XPW_aj6}upSuK#0lVie3{9y=^2phEaD|YN89F*k% zpE}gMI69Jq`TM)`os*vuBUZmUsXOgq;)7-6kK?{wmPB3_2kGx^cAHYhd5n3Y$;i`I zksReBjGnOu@MxcTu&{GmS1IG6p_O+`DCs*!6o@VS?m6P*grnod2|}`r`a|r`uo!2| zjfR^yG-D_r5icc}+XLWINU#@v17d(v!5gncdAvQ^E$^STy@R=)VKj$^8|^?Fkz4(2 z$FteG1|zaD^1EPdqVTtzyzJxm=;0rh21Sh4ku3Q=+zjt=RKGb6U{m`OX>up=f(9lk4`Yw7 zhubYup5_~f8c$~!3bogiAxJy#$3P*)=)=%LCL^{8`9AmM29GwVFipht09LI4_H`~) z1V<);2z4G%f;nxJ2c|&;L)|eb4U^bXlqRAGFLQwIJcdhc`!J6VF21p6rMTWL4l2Ah zo+;x|HG!|PHjzflT#j)~6<`;#8-xL^g~Www(cG&&^lJwOq}3dOR4{@Juw^GrD`&|< z#n)J-{oK6uj-xw_5hzOn|8*S?p9{>9S2ZI8Aqvx$$6?)%pPwaL~UAQh)?lk65C`_NvSYjS_;xt@|W0*VsjC3a3B;a0IEv$*H?Q_=SrJ=0mQHzab^vZ$8TzC}eS2Zb4c6w9OgCim=8%nhrVDd|=T z$au)1!*w{D>&f`w=5xT(W{7hL&RE54H#<9UoF?X}{@Sv<26>Kk`LfW^lL`&uuG8pM zh>)GQZx5Z0ktfAjOqqesMrBRs51sx8Mn{z0yQm{GS^qPX=ByE#lX71_l}( zY&s7~&yg`3FK4D4q5isnk=W-Co*LU%f+?03Wni5tHv-mF13xf?>zr9ilA=8}Q-#d( zLY+w7d9X-;P$v!bI9`P&U?Lg|_9UO-KO#*JMyhcu-ZqtPGQh5rBOv-ag$pu^o&Bn? zQ^S$kG?J%`G{|uVLfBm_v4nZ1a0)`e&s7*hk4fz((HH@_8JV}e=_H$L?68=7bGf`k z7m&_Hf?R2DK!xP@S)5s~g3hKr9f758G5!!#5E+C;08k*wWXvp>G4Pas{}TbTRsh@E zujma?^qtvr0A7nm3C8Z@uas3^^rWlzX=Lp&zjh+tnVm-L8gFzdPe4qY8tq4 zf+_p)^xa!SSi6)wXhTa;{+0|2RtA`eWc>I+Si$yRz7Yf-P?TNyjxoO1Xh>d8h64}$ z!53U5H&>v0afQ>vIOBiP>ouT!_;Mzd-c=o=QCVazU`ztT4h7KGligIG4>e56hqfHq z1R;VMG4;kq85>!`Xy!_71_Gn4*GW+-;<8$)zCTJGu0L?Ni`{Ps5w7Hcl!pT26cIEa zjKv~hG=!;&36HK=Tk0yM=`rZl(f^T-j<9^VjQt<6X zv>QE4hXAZN+VRZFnwP?Y0mN8{V8O2{5^`$OyUL$h9OE>i7v3ZBIxYSX3scec29GV* z(O+1cp*%}o1a0v2QBdg@PUXeU1bDx70pE3ZlA2QrP+A}Q$fh@H@~mX`qNb(njJgQ} z(#bW&hFl&+WDHs__k<13&0*XHQkt?Ah2O>E2lm519saKnm^27tTmHD5et&Q2EjzF- z>)Xk-uFrp9n+#K^>)vMrS5lXDX%VXI%+ROGZ&-GVb`w*Am2~?I&cl#&~5+xydq^+SO z7Qx9xcUcHBP{Otk@l*&EETbX-X+GKV>Fah0s`kWJA zpBw_@y?BszFP~_h0LPAYZ)=jLod0MDLKy20#DGQQ`r)t(_^)%NfGj#Y@V^(+&+>OC zgMC8vK!_(J?b_u%B4h(92nXcI&4vD*mykB^H&+KIF-c*BR3s2w*7+!Waw;%5M_QDT zixsxthO;5Lo^y`_5_B+lE{Y<+6*wDTssMSA4$$;pEs+)rN?jNI7{+;&scBOhSKnFi z`nQzo+3lt;V@0x!FSd=dTIAfBVr5w5&gj&(hA6rIq67Gl`A=ohUtw%?RLWo%7H`Lt zSuDTJ;uuAvbNw2#w5Ii8tfc|(N6w=)EM)9d5fL|mI5$jTbx$sY?H%GpT3>{tsEt&# zpq+y!7>P85xdjDdMw^24z{cl=H1_soGrtMGkSuBaeqIW$7GA!ESPn6uHBC1Sb zcPkhWNEt>Q0Ili7QrGox> z)Q6*fQpk57?4FS8%()%HU;U230f~8}xux>u84TrVCt4Lnu()V=)Sgt_ToWL_4u|%lX@tZhU0-^QJe++rzsI}Sc6Cg|3&ga}H`Fx&z$t}Wd;$z_QE(qL%@H9aCri2LVDIm54r5#V4G_34 zS9<-Yul_m;f_`DiUzU^EQ5uV$-)nBZF{#d)Iw@;&;+mOg-pj+nAQYm=h>c{OX=v`f zW=7`oJ!=n+x&|}qkGA-14ZAV z!@`IgCo$A;9tpAt9TK5#jon2_d?gqF+k7am0>J-sj-(*qV#om+^&@wY(D)UCS0M-P zNy1bKQz|iK9Q~7M<2O1{$~Y1Ew@z8me2Vi-^mHmk3`n-ZG=oK22Wh>wwJ%+JGuy~! z{V7&*+_#5@Tgh~Or|2;1R`~rQK%@buY<*~Q7kXGD6-Rmuq&>Rl(&@Vn4IuGr=YI@k z!r*loZzIRpe$U}35@1Cz>v~bnXKbh>(TQw)XhCOiYmnU1Nb|)lU{Z`=DQwb6CP+BE z5`M-WPBJ`4RMNnJkMfC+K!}ie_%XeMrX++JLG&s;m&HU5OX*sES79dW0mD&z&n4bz zSOrW)%g7{g7Fz<~FUvDVskpO1g6x>_$z`VWC!!JkLJka`bb99z)^a@Hrb{_Tvhvgo z6|&Rb4MyjRHR#7Q`<`3Sj|jqxjw%e}#5I949u3A#L?Iv42JsjQ8-&gvab%Tx&9$ki z$nB0yh<-MxG$0DZ_t^8w3Y2K}t-MgpA(xclLcxGha4Bjo#}t~M7QLj9fhOGthAj>vzYk&hJHp_wecXT?4Mn>rs?TG+#(%`~U%w0qzl)2# zn6ExH9ETtHSu;Na#Szo)8+@7m6!_cs_}qdY9xmY=`&ezhcGG>D8@u1|Kr2~vnezjF z--taAC0KHSNz-kK+i#nRHv`f8x$&I9c%TCD#Avk)4EPiH7}3KC`v0uCtmc7}p$?6S_@2PVM4J>Pe@&)~I$$f0S!`b^32!Wc zL5AL`8cMr&sU6W$gk=Vcb2TO<{+{A%0hc;%)dOlG?)!$pZ@os{;}k=5jG~R445r9Z z5sYai==N6mn#NI*Q3!MqLI5Ny5yXG)*#L5ZKVGUl zFyNQ%OJRbcCD_i}vHw_fA4-yc;Sy{pI>ADkVqs4sNPiSJ^sJO?|8;o}HOe%o<7Xhm z^T`XT*E>oQM30T5*s7zPi!1iIilp8G32wi@cGKnhM9zcfPKek~NcnXqpb3fB(1vPG zlLKc#N%EbmtjCT#5XWW|g>jbDlrXsw!`Qfx2M54gn94c(Ub6(tXwyFBHO}79bT1Wn z2<>4c$f`XU%HUSDRh%{sQoXqm-`~~tzlarJ&==EenhrzCy#xapT++$dS6AxjOq(Dj zc5rc2RY(xri-YBI&&C3VbrGAgM1L~W~@w$q53nCLd76S@X zMDj#5y>U#!J++kNr85HcUx!Wa~hXKDNlrU0UB4jvx;1H5$xH*C*s#HKeE-9vI7&H7qo08TOx*dVWqb*0d*W#nB-Oow8TV!J2u`GnIO0v0WE`^w}jX1`Nqq* zB#@8-vOKKUCo+hp0a9rt=j%N3ljXpfRfkKJh_O?T2Y`+%%P;`}WbW%Eu!EkmSy zw+7!RU?rV|ito+!6mIIZ`|BM9#$=_>Ffgx=!JE*Aj%Ha{!Xk|T2{$@_XV9o>DQcp& zuD>Z|FqxdLz~@)Ih#sla^PtM~f4(J?fxvGGezO`w-|~2mCq@{UlQC+i1eorJnaVPw zpIRvi*s9}bOl6=HGPfWT!s6KALctI-E6hz!aA?o1(O5P>5FpKS=Tlxx?Xyon8Umj+ z$Ow`H-;9V|3@`8yl9$*EdG|ro3P}BVT=PRgWU5FAr?v#@*2cDs_JT;C$3dw(j_10? zC3K8P>8;O^A|9|KjzFNu4ZDVR$`^V?@Gvq+%1J%@G3GuhEDT>rX?&7PLRdtE`Yt0N zEYI-WSuSC}xQHzhP*@!T{8h|v1SuiLJ%#q1;CD2Hh{nph)XQu@P?8l>zDVbdJoEth z;aW>^^6X>=*kkAQ85wjE#1Rgn&mk;0t^z?0$a<^c6wsB4EVEZNV|Ed9DNMfgLD`hD=B}$E{A4r5FkJB1XgB*4vLf@tirV_Dieyq$ZCDFpx`E zXi<5#_)%mj33j`4As;a{c*(U}59e`?wI3>OXtLoLIO>sG+I*&=UWjFT<6MOXkCe+P}_%Aoar%ZZ{YGF>_1gvP0gU$!b z!xn!>jjthu9vrk6f{Nxa&E#Y$Ax+?s5F$g#{3yk`dI9EA>$yyGE*Tpo^qD%$r=>f| z1x2t2oBqa?oLt0LsX!Kb*#l;_1{N-XzTBgKG$#8Npo7E=q86tNyd09qL~V@$M|kaW zUfJBot+(f!@;#T$n$>3lEwk%wQHVo0&RKL!*wC)yPI^ysyDs`0eEW_d1D8Y5@WA=o z5($sQlOyEVHcKz6!EX_@Nd9KY3ThJG|AV(Zz^#qUZy+= z6y;b}kx-HnMQTB;k3PVR=R#N1lt{yV7}=J$Q=w;G2`=KY;oy!EQrN&RQL0jd#-p&c ziQDR^Fx&FFIpo~)-PiZ~5yVp4#?T2PkWwzY#b&n%$>A+`+KyGq%RIve`;o_1hoNU< zW+jZ-bQJ7M%(mu_AiLR^?AnN_Ay@ zjur_z>-jlpjfx;;g^*sM!Mdvs6)is|hKceSLzZw0voZgw= z3c20adM@$2BN@A$Gwa_4XE%N-|FR|Wdzk-?!2s@%=xP2GgzQ`JdKE)?Bg@&M`C7;G z7~sHJIgvQn+m@8=YPW0B{k;|L=jZKqzEH-EEcg&~qt|CQF+UG_@6WXwS7*}@(W3gp zBObeW`NRI8ZOYTCdwqdRY5_SkN5(p`U0&EOFGb=uheEz{GHywa)G1)>dj9!Tqu{ zmE>3hnKa_XBq#=RqYl0$s!18%bTr}cW(yK%{bX*!qFo2-3qtG=ZSAiqUQy5d0NxDH zKg`P*Rp14&F+=EM#~z;MxTs*HC}6r@3e+X)DXmS)nri$8^l>+-s5@XsleZ?(7zK4yWXqe5l=)M~3#ITM5<%T3rMe!(@ z8LJ0laTuMZ!u?AJZFnlH25a6Mj&e-Xk&mYtPmenI2y2qK@zT*%Vj1&InR*Qhmw7pu zp*|ev<2awueqRVU{!pE-p8S=Si#0^46lN1DS#VLyVM-P)z*}i@Cd!SdOISWv6EIB<(8Ro+;0vZ3*~l9Sd~)rD?4FN{|&nCy2Q+L(?9sYIGNtCdZ|v) zJe53zaruWMn2x;-&Q*Vxl|EJA6##yy!~-|T(JKuw50t_j8F(Xu)CL<7a=dHjRDE_7 zWi*PS=V`Y3q|U zX;w8}7~h(7~?nBk@q>!)h$8;>;OTX}z0UdkCN@Ao-`=EmK0e%7V; zd8=$E_sl(QXIGB?IqTg0L!75$)xS%^b@Qha_{J1M%~|_+``s!1qWgAyV!vJN>yLE{ z&5f;>w8<>=AN}^W_v{`;nfaO1J$}0)`j^`gZpFlBUZnuHGTxMFDW|KE^m`PTAm9fU z#StI2)W8|w%b;z0njyvE1E-oRqF!DYCCR_rmT31IPg0Z);Y_l0$%~bqSC?&x#ykhT zDg53f(noQaZGx(uv@4~{qL+g9Q}MM?cedYg=Ormt#jPApj>~&;LB)v2lBmU1Kv<^fY;%v zVc!a~yXhN8zJ&j6^z8N`)9sHLOF-w}#-- zlk%#bpRF15cuS}V4Db^hCIUdq-7{b}^3Y4n8)n6o_3qLL>9}-&^JtE6uScbezdo+W z?vx}A=1fjqmg8Juq@Q$(awt!s`3#@SwNEN9**+LV#x$rG3Wif?c9u@)^3j#+MI`ak zb|$0b26R}JV6e;iZ(-FZQG#!@6P=PJlD5>lbJsaiHae48z^#c4QHpJ01m)8+q1|KRdk1ZOaPLAiix__^7pbExY@+uP`;{UqAoam)Th`!h zOHt#@;fuL?F_sRXV)<+dMZd+oQfZ-?XFkl;dbyX|(5fRSc0Gl7Bc1pPfrIbve2g00 zdeaArsKVgOMhlvW70J1+;raPQL`$szt|kViHtr0yTzM$->m)e%`FXA51t&b0D%u!_ zf@qk_E6$8+Jyx3cu@rrE%1PaRJSXJ%;-WbzS#GTgURWfcFOm`tzxi;tgdjI7hH0ks zf-q8wk<58gY5GT!Xr)))I{vj)cJICV6D3QDWB#~9RXgl1)xr~fX~H8*hK(6^{cqc0 zS#Xnx1}h=D2q6H|ZraG!r)+knD5CbA3k|jYnjg|nDT{WDtl}Z5Z@3jF?Y$jca&-xm{qRJhw~XVI2cGo1n7feI_b_Qd_AH6z^s zJPf^pB%}TtXa8V`g7^B9mUUn3dIdgAj0)TSqDo9;)3QyILVYf;xDh{HD*Gv%{nML* zcCkv4+@?tMmdF*s3fdCEN812HsNs1WC|+#9J$FE}0I{pU5vFruHR=;0?N*^CQn-i_ zBW_B+ZN3F#Sj;=Pw?UHEO0hGtA+t9XTOrn z>zP++MR(`qaXihvYgS(5zl9HFjSB41d5)O&2U?7C%}1vG_{PuD*9z5zL}yBuEA;C0 zKV?5ERj(*@t&{_zqNFL^qa?!|DNH#Rx)phuDb<`RH5e&)gu{Q4=P(T}DNx08SUyHi zB1_tM!rJ^!b><9pQ5;(lQcC@UEH=Jk)hq7!GqN$I&hhft3JWgiIssRM>JcjNj!D!u zvSqrxL_oO3M=~YPm*CZc83mc6MaGJ{2{EOO{PR5{^8D9OM)--W^00)2Qa#-9v<#_g z(4b=;p$+U-sma91g=%__N={5Q8HWiit_=)0>h#i@b*3$=VVD?OqXqM3M0drZAPvi-Ef{-(A*Npnnzf_d1q z?cU(*AZ)_1MJWKw_j{A`o-DXQ`-C}rhbQC0o2b0*-@+H#ZwZq^qbEHSGn3bA`3Kye zX)ldhoEQ*d-_H`tM*+QzyOe}bs!Xd1r%H|wMaR)7)w?{lMyqkLwzTidT<=OZZOvmQ zgPu<5Ra=yMogUbQ6(%m5g05mp87)g3R-{4QF>ZV{gVfh z=VHD8PVFlGs1>VMf(53jc8ZsS@KFe{91pXTl}lFhHA$v7pdvE)=nEUyd6>C*%G?EI zinpvHVv@`L@hCp*h0-#}Vlb)DYM;3ut? z+yln;#LNX37yIfjY=pmYn69qx>B5I;EZ*EVEd=hfaOJL4LtLm@=A8-?n4j0QoFiB> z=^NX<7v@d{U%fi?o?-KaG2H+Q=K;ftoTp|DsH4-o-{Z~MQ-QqrzXSYF|H$i4~vg&**WysXtEiVf<3r1+k?lS1Q zK1-U5)9aAQ`|WAUe*9H7WX3np3hCHFyrT!xW7+GAmgZg`%OcLd(#paIw(XvXy{3Im ze;@wz-EXT4f0(I$!NIWmL%e?4R(G;h&0b0KN9DBbm(8iRqH4{UIO~yK5SA4i+3oK} z78_42r?HsDmyXqEZXCn*u3mF9u@-S{nai*9#})S7S-sI@g8S+Z*7f_|8VIYpZoKK* z3gruPmc?00QLJj#@Vl223B^*c{8^em#LANQ6N%%RQaiOPa0avC@bzVRlM1C2Cz`u0 zaULXL*73&jl+rGFO>6YODq{8uR5UPW-|a~e>f;@ocW;Yq-uK>DEpaxq_f6XJm5akY zyBEuJE11e8-9WCpOLec5-h5ww9||t&RZP4J$hFgcR-M(V{Z`%n z+1>b#uNn;?j>BNr(3xp%P~lG0^>?kb*%y^VR7vU_ajr;0l>3pGymw)Y+B&s_%fc@E zgT_faEw7HI3!%pQwUTBdm(0X)*$D5s@AZFK0Hi6)(_$t!-+E8TG#Yts7S$?`))~^# zw(ckv2{ZR?Yl`Z|zF0QKF3p{y-*+g1VT_Z~Suao9sYdbv=& z%(Gfz`?fni=*0l-aJ9ikQhd+rNj9fy&W8`U5aQ`Xl1p0K)A-hE-&5;RrPjOj2okFt zvHJ33=B#fxFZ$4hpz$HY=_g{8Qum5pg{hUL$0;B32Jg{; z3l3s>kJ`Lf+V=uBUm#_r>^9^rm>F@}m*EqJl^fu1@cDZc-;xikP&Db@s%hcZzP{Ux z4Uf-OM{0PN21~t1&mm#JEq1{C8#g3&V7)czmOh%_4F^5&QfJ!MD_P}1pU!^2D!hM~ z%F@JZz#l&OxbeT-lYgi)XQov>Nk3ZQ~V!hewp< zKEO;Z5~#ydYn0$(&)>fMh?I2tEhm0NC3uk%MD8}Z=J-2R-Asx5zA_!=YoYzQ&O&Bf z^bumcK~Tx4d<>u5kQab4%#J~Wv8WhfdP!nR@$35%O1S{aC_z?h;|V1_W*9WB7qy`$ ztcMFmFzaVaYjistTSQn8w{WLBNyG*beT_3!m|!YNMYiv)RYq>3EQHm&rNP#qIRkjLkIMf zio(jql~4$`#l1RB$c*b*Cv$VbogRNpcmLyc5~1sy$-_0L5(6iU(Lf4iU6Vo(mr5o{2T|S|eqW`xRKXZ`G!9@CVmic{o;RH4peWhaDfA zOs9}Tvf^u&C`~$i;qbEiR<(-q(ZjS=4$?~1*v)9MNEjYh7#=2+;WeR(R501}@w#GD zp@aoq9;HwHoz{Fnmn1Ga8nWf>r1uSqDtGW&Z?aQRc(A}eD6!Z58G6tu{;tApk%60x zXTs0UJjo^QURRFvkF2s^oR!nvdqW+x(PCP$6)iWsSF3ZUI2)7y(3KRD(uKQdl~2i5 zZqlv}BIC!!V^m2keC3ajQmNofL)XU)ljf`%Oj^a0E6z@>QdG^n^9g4zATVVg@M%Hbq4#QY>gBTcX3=<+d)Wd@vTn$DRpyH{f zH6dFmP&!=rOoT9z5UO#(7;5nN`$|+Hy5QMQ7kP{bW8q zRX4=`f3M8uiY3c$nfyR~?qC^r27Y>w2%XO)X;J>}y)5-lc^8^&NVw)J7a|C3>b?uO ztr#}Oze@@V7&)oP_lcHxI)#XtV=+x_@98sh395y74*J4Mv2RtYD&P6Wr7@~da%BU$ zY~?-pj`KxNTM2)wnP75J)WFG=zVkG>Qt&})oCmkiq=Hk^*;Z{k>WANdy^8N@nf#Ux zcev1}n98J+UEuy@L`U88t2&NcYSdO`!gdXS1SEb1Yrt*vb!vnD81Ugk0_Xlp>D&Y5Q zIL7srPwlyTpq$iI$W1o0{%xnQou=2b`jEMy0$;a_#~=NeXW&7OgskZ~pPMOto-Z@* zGX-^lb#+P+o)oA@aY~nJc_n z-}$QQ=ML{RXlD3-`hc3;-t8!HdzgIoMt3d0hO|_>5f7&Y^R;{H&mDv=<>F11i$3?N zdD?NC6nssL?B@}@1^5s=X*hb|`*Y+D@dl9_XF6$)tR%5DW7M8p>GCD4?b2+FX1_(}lDR`{g@~k(Mi=KB5&-?1#$f0=?oa+A3Spi(a|J)<4pj z1HEQVKZ#X4Jn*lt-m3d%hFU70%C@`tyVlhG#yv|#@mnUwt+U(e4R~9q3uX^I>~&?` z`WDI%bXb^9{E>c|V)@Jl*hqatnjS?hsz+F&>i7XXF2&^kL4Q*61^)4~36s&rojXdu z;v9U7HSd$F;(`6&R4{1%i5I_JZ86NW0=m|-)A#pzxgS_+K?xzOLhn*ZoWWBHC82g! zlZT>gl^Q_mT*KhwLKD5RtyjZxTD98vR@|5UaFoW%v2;v&JIL&3pHah0)l|AoTRms!Un!zUy4EH)IW~mL!gqO4lG8NfDqB-b9ks#Zu z7l5j0AC6xn7pA0Cg*q=Xk85&FgTk<|H-Tb{%W}Q1{sTfmf~4++H7!nF;TjC0{`~xQUjEx#xWv)nH}Sj;(lpS9Osd#%rk{l&eTv)Rkn*LiR$wVru)e;>x)ZTi#y z8^h0mmQG@6o(XOR?{_MAMLlF*mE2THAfSS1BlUBp!?Flu6h1}(_s)H#k?DdIvo@iC z>j(Fq;k#8<i*p5xMx9nCminIzun|p{76B_;#xvR$Pc{`Q4A0wpR2 ztO0J|Z%P7Kt6>x@8>46VF*nkrCuQ#8oa*jBI!eX%d^!FibF-Y)N~+UGR>624h*#uh zWu#dBe4r)iL%6xtJsYNjaz2(^r*pv{>k# zcFM6N*z`%fO1iBa_He&nR*jYcgn;l-y)J*%FZSf9sEIU}5z}^??7hUDPqxtc1!cD6 zdu1j?J#u1V@((sf3r${U$vl-@JO-sE@r~y{S;irT`H8o>L!x&5eO^Hy84@u255wC? z9;a>EG?nTrzR`%^W8Xta=tLRn@w~vfC9?4DCiy_^<673Y z2`Qm)2zrMujq{Ff#3sy$T`Py(X;VLaE(S@B89<#Kc>Di1{A7j!si_O z;%)(dzB92}(t0@qed21<+E@8|^OawEY%;G@Y8fhbI%r@9g-jzxb>4=wPRE*(i2jHM zRKz_;;p}=xkM|>`tMG5u+S7;RHV+MF(9-3-N~VitkQa$-RNW)_7Nt&$vneheovF*H z?YChdadhWZ(W7w9gnc6wN5}&@QB@PLxI5Rr@?s!}kiQD?iEq6H=TucEYQ%K^+cqH-u zty(1MI<4cTsV(&C(>$kMF7^SXrbN-q$3g1n)rM`Q=>zB8+Oa%WAq72t1L_T&InkUY z(b-gEX%}4QPv;TMlF>!V-&Tey5i}Pu1BYAJ=)2)bTrqE%8P|Jo66DWM9}9 zoF2rx_QX7=pN(I4qTqZExR;WjRvPBX1v1%o+8{K!uizQuq zycs99)=eS&t_a?WlY$2;M%Yt|sWdQZ{6MQUqm&1$@(>c>USx$_4x>k$*EMdBgfWp= z;t518&?&)WYpWB+Rsx=)nMad7_NBCDEheV7oJP#YH6Gndn9^L6)Hx(E3Dh+W=@?oU zNVKDSD(SvB%4A6>6qe~jX3n8iTTI;*rXXsNnGdI7>t&(49Z!b+wd!3`@f*EN6=dN= z=y(SQ{pvw+JGp4j&O^hrSWJ2))h*9(R+#@t*u__zy|u^QgC~QTig$$*Ob$aZIYf!tfic4 zivi4Qi$HzlBMDPEBl1ioFL>2*INlQc!Q%Wn8*u~Nv1nOL+A1}Y(2)5QRyl8#lE9at zv1uwr(x;2XbzRYSEujukMJV4hGRw8U5`aMzO(DsXduC7&$jPr?z1Q5BE!^t38+?BE z7K9m72vc#q5F~wW1IN5)+6NhUgmIU&WLmwl>t(!_sf+ci6xkQAKB_XV10|SY6fwT# zK?gepek+^D{a%RD(AiciUit|)+VIVdDF5d!^#_T&Un(EpTQju1f`c_n_jEUI1?_VO zY1M3hVOb&Em+U<)dgk1t1tNL!@CSBaL4Ht&F4DK3`*wfk>B~dM1RxA^U3s$og?Fdp z8LV`EWk)|Y>!%>Y=)WE+*GX02nd?l+mBScl9~DS>;?_YHp# z2=n!1nEA~%|9C}1)#gz@AjP&T;fhY50w(iSXiwgT-Bykav$Ap#XP`ApkJ!2-4M?-a zu*VgXVSOr*am7VJKW@m5#6xMToM%tmTV7IUS`)KDqcPG*2(pjCI=Zqo4zbK$$p zZG*{+k?{Zm_$+EXJ4qcd6R&xYDjW&8hJfgD{VoA)s#_E9U+V~WkDmlt-LKp1f*U<3 zDQZazM(KEMSU5^Y2Ga?eDwB4S#0`$s+#BhbNF?2X%_tXYj>t6SeNB~QkU=1#f=o4s z!?`M_s_B%v8;NQ?5=fdFdaacpy6Y|K>1FsWTZoF=CXQYc=C5fgr9~6%+rr)aDmV%v z8&{JRSjw|m4?fdy1no7`uUCBU6=|_6RH9ljgAeGyHe6`wwbOM~Nh68M(u zu|#4{Y0ped>?Q>tIAnN&CFnS%ly3A`JgLp~A=b$C%Xs7+q=|g@*#ASso$kEoH7A3f zY{dityj)GQ+Rr@tTqJ3J@9uz}eazr2)QvMy90^+mlg!{1X(o^q=EVCgCggl?OjUeS z9p{UWKiHk^1USvL@0j^^Jj!?o9hY%r!bYl{V*K7`#5_U2f0Oy0flhSfuHT&i2<>tn z@8@tQ)9*Vu;(k~9m6nD{33KgoVhvAJc-Vx9#~m2)S?(PZj6Ak}@ZeZkj=apeLn+rh zg(PNM=8EO>vFeyke!Qu_$EfH9PaXyr zwtH92AogM1P$&QJ!!9G`s;$pH^|cttx4!`YpAnOX(6-U;1mZ(T=H!Qm;m){tw^9xJYoL_{O(Ig`*mY0`&TJ9^c6G&PG0~Jvxf0!(a9dgPg{0< z_k+zhqs%E3X0Nhi{)A`JX%|*gy_!IX5KxF>-y3w$M9Zi*b_g$}ZUJ`Wm$^Rzzk6gWOox zHhn6|$0WkLntCqaCoZNQxO(bhfnrx;ZGVbx^jtHPwXRp;e-q z1D0#WbBiuM^bki3F+HSOxwLI^uHh6uB=u>mYU|B};CId>Y+%=rwUKw|s=If&=K{^} z%sOQnK#r;LwSIgPDMY(XmYRsb?SP zIZ#`Q@<5KFsETUx5NWb9KOM*iw_I#NMv}s_o!5nz@(&@dqS@#T@pTpEVi~LoJ+Sge?BibbdO`<(0l9pC-Ytl-Sl$4#!1-j3mS#b|9owb2ps%Hu5&V z0|DQyP{TeSr59nUVg}15S&&jB9#>wtL~b;bzJ{6Ym&F=RH8B`Tq8WimYwiG~KsUI- zgO%+5{pYoKDV0WNP%4o=@x8gaeUP_cBT1h|N<;AC zc-j9e4;y!aOEaabxSwc)((}e0y{8)FX5e|6EyKsJ`%a4ES?&8=U!{B~^&6(c+_~G^c%xKL!_7r|F95x-fNeF*@x@9nOve9cZj(nZyGCyWM)Az2}G27!H zYlqVWs&n8sU8d3vTDNgBIbnlBv`w3zPp$%xhfy)|yi)#9C-sC^-y!RSDALJpj@UJ+x7OE-K!$)=VS|KOk{u-T?D_HQ&IV;il2`30ro3_?Ta6w``!dite~l4vcvlp8>e?rYZWuI`9mz|QKc zo?6Vm*P-9joe#IfjoAT_d|5UiiS?s$*vuG;A6r~_j<7Rbn8X3hh-hw;y{yE)wJ=eBoON>*Xnl1xc*{hgDH&n+9 zl^h3E%SOJJZ1Uf)EK#9MQho5U^Rc)NyR_kkNlKe^NdhwkCT`9fRd?a9A|EdN*&v#G zlh+3%4JbFFCVle&+ZrKstKGA{h)8T&sG0-qwv@TBp3Q_AO5P^X!fH~xUqcGhih)Vz zFr_DQ`S<`1w#9t_u*|qy-v1izu~?m17QHC#W7J@Fr`?o8L87NMBgXjByLN&&jdIQE zZkOkCWdA(FvZLQx?$<;%p9c!Z*TU;xZPBDMMhimoMQnzL8R;lSCpVOeVV0CvvsWcs zLp~B^JN0VV6+Y07-*9+x)h>>w*%1i5yt(CWP5;pKiI|LE;asq~j%G_*P^U)iY-s0_ z*~(-tb&US5uW|N{VH!2&{y2_R<~N8$_W1+#U9*Ikn}iy<-P#!Pbl?l|%DeE6=O*#1 zJu6wUnCoc-I8Yd*78+GaKR~KPzNH?+zBuz;S|TqY_JyI6Devb%A_R3!K)`hL)m!%% zX551&qz(?QS)O(^#C_Tct*NY-B^;0kk=!#G_ABDSL(!>Ic;qCt7H8H~$zoU7l2KR; zcqH~4;o9X^u%Al%5S5@vZ9ezTo*3DFQop~F5iml{I+tezcgKZ8W8D zX#7m7I&6CTma859;3XakNL`7Oh2eP*b)b(6QJYE)(xxoC1QDD#y65rdW>h* zVfG*4TSjfRn@u(77JPasRhE_5I0Ko7nlko{IFwS@xD#{%! zVFbZTmgtm}Ia#b88q*CLELU1;$C0n%k|eB*>YmS-A%MR$9&b134Z+7HezL6ASK@e6 zm_YN``K%VK{JrQ2P3#HJ2YZ&Re!^r(MtW0(lWWXcO!Y=x5FMw!SI1$OZzdI&Kx)`h z!cEEHK~;OsnTwjZ{ROyFNX|vJ_i6T&bMz#W$GF;fb2z*+5fD;i(xg)BQk3Gw`84@d zgZ^9%klh0$+;;!Ir-+LA64X!@r3f_D8X_OtH=^QQ$P849vPDWRjL z^VVpDb@nVn-%$Gbfh;6OzC5=-YA4I&gvtiJ&xVeoh>E7n;%!aK7ED|NPzr>pjAq=F zkp#8<1~J=1uk&RJBwR#@8V^}H?^XLwWFF|$WZ{x!-s>02Ju_@-#_x|ai7^W#JtW|? zpLY){Yv&v&c%Keidv0rW8k%BQ2*=5H7;H46CVL`n=Tf~!)4 zJq8NC&%HZG30EtJiS|`!Q%_oCR7JOZWH8u`>>pS^}@6_`S zvU<|Fgo0Vy>x-B8Rde}$_#qG_A$NC(bh`GDqmiM;MXcz9G#UT!)qpT%$!yO|dCbE; zF;8c{z!s}%WpAMjn<4>1W^>O(H>bFT7#wwNyGfh*C^{*UOU-SgfA@Id%x}Vxk-l}UXT!2)5C(+*I{>)mL^7|^MT$rGFY=qL;n+}f*sv^4o9Ow+^=fu7_*HlRZ&&f|>Lrs}rBV!7#2 z9abK@VjRL8@n(rfFszYhI}Fj1JA0K-APnn!L* z%^1DkYpuO^=m~>;hMlsLwE2QOgqgy(w zYnARPn;cA*eV)tr=x?x|Z6=zmXIsQzB5n+oRDK~w>HQmL5A1GojaJZk=dPIbBWx_9 zM|(~ByjK^Z)i>LZJfPpr`p(4Ka1JSQ5g-0y-0#M@as+2<*kSkH8A);4Jv->1-hfuP zw@?vwMnUN-CZ3IT?{@>e)iB4wXb?yzQ-14CuR`Xae63`C)jO?pm_c9AU^Z+0C-2~o zPo?q{i)y=7lXiukbtN$#1xmJ2+bwaVlGo<+WoIXnS}lMD&vv+G^&ij5QFoGpPr8bn zu-V78I!C~Ko;yn+h8?$c%=udil?O2!;8W&uT@Gb|IdIUGbC1<-F!%Maq}|6-gy3h- zhEWV^jso9fYXnjLq{q=ffs7~>Jm0Dgm%8&nKYJ$BAlx2vQLG~2<8pY6X#GlIaJqw0&Ur!5#4QS#f}J-#)9jl8;Fk-lSv@9oq9Wxr65EJDx+ z)}$+L&3|8L3So0{*di41+xp@@jZ&n`uJ|(9uPb4pH#~(}v!$PLEH-Xd^2;$+K3c0I zE!3frVet>i?`ta_MafaSI!WHB*KBchE6J06IB9OI7wu#<%A~ehlxZ}$t{*Lt&0WJ3 zw(k=v;ngKYG032X3A!0(4_;ON@}^bd(fNptBa2YY^xDgVDq_!z=^(R}y0#LP*F9N9 zyhp2LZd#e~pC6We<*akvxNylU)X5YbnmX-b^^r=%s$LfQD9c)XI(6S*d{hVfuHUH2 z_&%Aefh+Q6q_%{r(*qODO#?wv&|92YK>^rw3TEEN!YQt@n5&j`qgMXi+%ocq8&=%i zHwqjTQq~L=G`MV7`tuXYa@q|`iM3~s23|6if6(Cy_~k(0Za8f2_wS$Lf)8>i4nz19 zvAI9Za=_x~SJI|)XRTp3%z4a&fza_z!kFB2c)95Lf)92LVZ2ef^(?0|a{T#aFGHU9 z2S(WSR_i$roc2_=aD(yimw=%3Ya@5{R<);f_4y_{5lRp$S2JE2 zEb;Z3+-M6GcQ2XQ<-Sm(u|g%D^tuWFVM51ty8o@8#i^B<~9?~dbk^z;2}^-%3&5wE@L;wK@GKWPtq5}io%JdT`w`* zc^CR74(llkCzr=|L8VzNxhe~_FfH2)W_zsLU_O(gN>=i|jltlDHJ*MXMsIHmKU1*^ z6{k(-+TZjHaaWlUSXm7Hv_VLcCQ#GPYepc?hM^)v-`$!a`@x$#`Fw?55CcylP1`}C zGh^0sY0Mz=#Uv?frrn#s=QbKxj>C~dwR1;>;1dW~1JO{oQ2;leGJLf-xA~~y#h!UY z77db#046)qXHBVZ>KF$)1ETl=#hXq?epXnf^w4CAAm5x_-)wB$mGC24} z2>;uXgic#!G@~!Ca2P3PI^|m47$+~K$=xa!lW_$h_s!NPa&^7#WxOR!y1w+TnC9x# zxXlfWO-5{z(OmWB@&_GBT7>*si|sbq5L%RUhZ%0P+uDaa*|vmj22bgYUYqTNt;+Et z?p$3Ym?O~V+84F#mL^%VBEX*N#$NX~zb})nm|>sEXv~7aSMpjzpO^rLJns|MQGc5a zgQ^M34QT}jtP`rSZoZ7H^i~^v;*KzAk1J6}tzY%7hHs*hC-VB&VHZ4Oz%qEG1R|1nu9KztxSaoW&#-q4jYiS>F|{Jtv2Tz~(t3BLLWXv*OR1jQ;GU^Dav4jK#bN>#>j zHB3ph;0`wTw+Rce5uPx;`?G}O4#ttPhu2}vQ}4|UM*G`?+0x80&ogitfYhfNRR+ww zw1F01nn=z4VrJKYLikHA#xf(AAH<%mh-8$Zo83Oche1VXlsG?{LZ0A)>|{`@NpqlN zQH+6QXXK8>QUYCjie-{4E(hFvfoHlx4!(^ooF zUEN=ja|`RgMn^n+<;YCQd`5lesuv>@f-shjeu~VUzWiDfv96BWWUMh^40K&hLlfm> zbfoj(z+69I)j%EyMw7{@ob48)o>c5sXK_u_N7nKW?pPG7)(oZasjnKa#79k)8q{PM zs)n_JG)$Y{8rGH>HSn7#3w-(LS~iO=0ua`+y-S4k)xiw3c{AI!Og-U2Rl>m_-kL$R zI@C$d0kmM}zV!=J2zq{|*ic%fb2epO2tCcqhh97OP47|#ojN<3^VzR;F%=%J4#&b= z2E{?zUVggSY`Um4PeaFaPF=4!Z;-a#9$ihD%j-85$F29vE!`cP!CsrIMGfbUgm*>< z)kO}>LEMY*vh!$fwrIDLhP5s8joGp7LC#oks*-EH%VD&`EuoA$S3S# zJEg1T_DxD4G6sFou2c!Hmj^;pu%YFFP)-tEQxoGI87!l0sNDed*g=1JB|i0?-l13! z=<4>iU$<`6VHAQ^GAw~-b~*LqeX!kRX6&To`nN`eV=E!py|~ksB`ua>>7u$wi2fQ# zlh@W@AtY1r-q-!^kE?XLg``pM8-XZ*&Q{@!T8a>xW%tZvnRS@aRA0Q^?#(ZpIf^$0 zG=?Rx(wO8doI9hZSI@D{hu7z;HD#R)Q(QO5&_$v3LvL#>D~K`f12@o{o=&}EqyH6q zp#I)#f7B6KW7ryTIH_=Kg;yFovUz;~^cngWV>bTsznu#`G-cI|N4V3}dT1 z#@KU*IgU>;WD^P!?d{)Z%l@|6?G%7CjP=@+0oR>=endn39k_SGvR}@$De-}m1IKM) z800Xml&5a##`}0dG=Z_{eW8=;t-B!82KecAU16~`DVnX^UD4M{zOY{U&DMRL9b2Y( zm(j|);7LLS9iXd>=8ovW!TH^Od|DQ*;7_tyBbasYDF9fI2Eq!@%D>XbU`CFPH|#mG z7-g{1Il4+Kuv@yH?GP(&8IdCF>^aqxSr$xbcrCzL@td_SwA!5AFs3N z4da=68NI2)*qyya{;Cu})QRQO0Yu$8D(J%M)Bzj8*IQ}~uG$pP{c_i{m_P@XiXITE zg%0UGjyKmug)xCgg|4#3i9cd_H-?xz)3wq51&Hh%4uc)fnJxM#a{OkLe-|hqeitZU z*p42JB>ZxW~lM|3yV24ydA4gY?q&#De>W?-iM8CUOxGFnqgj>g`KE) z@VlE+SwD)}Mf5^$<9L+BQ`ExBbZd_wt``n-mq@}Es9_2^Xyt_5lpw5(7*D^gF}Zid zOu&;I&2R8ZN9dZV{)FY&&NfG6(m%@@tPiJF3OD>oH#)8g3rR5eDr$N>h`z~_KeEmY zvjjP0PhqZ)!K>&SzpfA9G+VORKNgHo7&Kp=d5o~*_<$e3CbdB zuCtlekt`UH)Z4_NZ?AZqYrOe2vfAQEI*6&?mP z-}Pi9M=6UC1Qp5O7|n^U-xXwCSL4?FqUsgoDoWP7L#|_4mCX#RnmC$72R59Vf(4Zya{CKO>U%ZA-W)Pdqfok zTJSsVI{sq^klWuf ze|lTB(s-v2sMSJzikq~k(AU|Id}6quezrYBA|J}gNRY>JFqOpS?rbzIQ&G$8P=288 zmPmo^8XZIxMetJ0>skG*y8A?140A%O+4bc2pK<9$whfC0GwvBPpo6Ks z`(LjlD4cC)a0EQcPxPE!wmQ=m7vLaY$&K}HHR!aGZ1&^Uj`El3#KAcn%edpM$bMHR ztBv0J{Q0*YteW*I()^t7$|p3GuN?{781H4YS9^R}OGZs5L(u%JY^QdflSWEMFZ|rI zKbxH$c;L+2qjlJz19R&v^Gh=s7vjd`XFlsf-!HL(Ck}tGg4#L5p1De}!`cPXbI(Wk zphr3$1l=22%ZQh^ABD^Xad;VVyRF}ta*w~Ga@Cn^Jj^R)9U?y#xAuxVzL~3bD{b0Y zN0blq(s&reSNDq4awALZ524IbyDcX@82Ma9+L=8S+$lEniVgS6(ij1hB&eGta}bkM z>^X6O9klA~h!5jFP0*fzJ_$?!Qr0A}; zcv09aw`<~VH2JD$zK>AaV9BBiqHaJPpg+#(mF9ht_6z`l3G?3PbV3&JVe7UVBqr z`Go@Y&u@gmjo{Jr`m<|=lXN@{7|4Ge+n2YZEw;m90cS(Km9yPP;TkyQ!Q;4ZBHyc| zq6s9X$jSv`R&vJ4pp4S04<{cp-jQ*H0$WJhHA}411gfbX^Mp_wTD|ShYbY93qPSJ& zZ?c}CL8pA9#d&q|jdL9D{ufZ@&Y1@%NgY?(0aC9k z-{A~l&%c&q8OBvtb1#?GXie2F;oaQl68%JW<*n#qiEPH;VC77=RjGk#|Dw5zs=AFW z9V4Cf9yx1X2v8t9%Xn<29$Vjwc%LBnwW!$I5B4RqFFy7Jx$Sz76I+kw9qY})Daex| zd!2E^yY)jxR<9T!$uZNZ31u(OAk_y4MpZ*^#|uj0=xK_)lI8b0sf?XGg5E56ZY+qegc9!vu1l{@JC_v$sR*aMcEQVfle(zdwMH@ z7DUwS;I|>xl6THCwl`0J)&SdW1vGS5A9qjbJu1?e!g9{;EvTH6&YKFWhgD2}n#^8HcJ!n41=9O=oZs8dJ8u(%^cXisjlte`61^PZFa7aa zXeo`Njp_?Yec|!a0gn*W^@JjOu;Bu@CmV!niaWu4rJZ16TzbT2y=Vwm_SUD?2?o;$tFGQsxWn(;9Vry3w`2WuB5{ z)KK#H6-Pb;isy^XF758n#ky{kT8eab(bktTHPS5e%x`bGA)@$^0^W{XGR0@snZ4SN zorcbzd8zShgjZ>U`GA~iYc0cJI-1-Uw!LiCS<~q=v1}ybxc z-skP!+DG8?R`3q#nKar|hGEuy7dm?Jy%`&{Iw(m2W0A|=qPX}TaMmV#>?8c@vfa+;$X8k?D`#3-q z?FT;g%hS__7z&S;Y#(1roTrSiZ9LqaRIvTP*&FFR-HoWWN9Cbvy^*_fr(>04rzAr~ zE||$|M!Oclm%+Jdm_>={&GpPmxv(P7&FZKG)f%X-&y*FzGUUcKuSRG11}^-YxzRLYmLXV@S;m-ElKwH6Nf!drthHKGSJ0}r#_U|O~TnI^U+ z{&Fad1$bf$vqAT;gw3i;1zqV-TeHntF!;hAm~U^gD?~1SXGTwf64Sl=@1!>Ss;T(a zaBHoGf1~;~rDByq!_bGy(Pu$+&T60L=Z}*bY=?V!ss;7I@q-wjtd?j$LL5as3U+mI zW>=ro(w>8yF=1Q+vn&8KAkWKneQpkqi)yZ`wH6J3i>U7S9LIbOWLyU*)GHR6c!?j> zOofRUa|oWdRl}kZy0mVwClRX zT;wR%+qw{yV=fvZvl=bunz#vs2$4{QZ|(P;R$lYe(AtsZTNm!f?@3=3_5W%*kI=!A zlgb-$KaG(xAJw9$?Roo&ME! ztvLya&OO#3`+18yO-{8khOxZ_^3Hi%uZ`+xjcWVfu(?B5(qeUyWTorzto3(CS@{n} zDC%V?3pp1oapKLv2fd`=$qV-PB^?=3u$Z1wWCLw1gnN9>Lu%JxQh{%iPQ(2x?oWW+tmDrwixE z82aL(IxX&(CD6s#WCMuVHhAQ27az+`%1ln5Zk89*gZlA0Ul1 zU2`oJ4S}UUqU3HsDNIqo`BPpRW%gca9^akv%b{>t92}2pvYk3quKmzeU$vE&&-7g+ z!Pw$wz_1g~#QW&rF4&<`R%UsI*gNPTs6Dv~hL}#!;qvx4?lk(e(Js}u#v>!A*+JyNj@BtbKacVc z{0X4qw@|pQEN+7bFUqPtow3ng`?=Zmd%cJiYwzHV&-q1#wI=dU-^muxFL-?_0&e6P zR;Wxo%1*vW2zjw;KATk=(vmr(C7%s!(|IJ8zO#mGfRT|~$u$HNLxDR}>}R7AUV?Y% z^H#PQx9)9S`(Zf=D11x{B}myH=6b$Yt41LC&p)i5QerPu=o7pa@4}rM5JSL4^H<19 z_qxI8ng;h(w&!tTo{vWNa_N)pPLK5?s#V&aRnx51Us2U@X6dnx-RtAE4v(nfC3d2` z;vH%^2EIRi9wlJ&ivQOUnb^3jAY{ADq|4A94?dgXX;@`R+VV6-W^;OSD^4PIKF%B6 zsyiQ&NxtmEd=jMKfyjL{gWkG}cIB|GnB4{d7Wk7w?Av2HaPkcFSp)j)!b4%U$L86S zIsxIN=cK{Z=;fC3AdQF3c*o5hbAAm#I(xLYjzxHNyC;4=aq^FF>iIS480gX|sP1^_ zuJerRI$*gl1_0>Q+5=s1xOYjA%Rg7Nc9LkD7dq)FyjKJWiM*o*f3x)US>f)nE9N zvO;%;EW0^QECL(5*`Vi{Y+ieD{oYAv;FhCqtIwxHAS}1GPXD0^#!5hJVs+1_#Etuf z(C2qSx~_(D`6NlFw(^o(Y&vk~yuavV^y$mb zu0G+wpbs|#$S+R2+0aes z3*aK_2}yzu{^Vc0KYeZI<5aDWL37LfeVEr-k0fSJ9C8DNo9y z{cg5o-kGCpv|qmP_ml1U`RRsJ8R6sL_4)>3xm7(3q0B9y7pqW@ti z6Z9t{MngT2kLco?@)R?#3{n9i{*d`f=+X4W+Wdupwaaom>#oq%`k)Mjxc(w-bS>Cr z6b&_3Dsn+IR3l`(twH+oNV@OkvAU-tKG^4jyXd+0i+%kwl8mQ6O64f8wY}7fH7jBe ztBtT_2rgBp>+ytY7S z5B$1gdhdPu{_$9Z96s;!^1G-7u;)UTjFk%tbH03&b4mEuKq7kiw;rZUt)2|dyKm%e z7ge1VHCW}_$+NdVr))w!pxmWc(i_#)Pn$&=$BzBvE)FmB8kQQ2Yu5xhTzhe|m$?Z= z$sfP1C0PzSG(q+jh&=n*o4~iR0#_X%_6@?PE1cRtDEOW|eLhF_#&cAcCVkc!zsL25 zus*+)Ac3)+kNE6ac4k_42kW!4@7W6n?m=huQC&WZmQ%t=k zNPUe$d@Q_0uZMS^M4J_9r_V;{_u|p~;w86&-s4f-IEh+ouU2_p5#O`6`+C4fYyt$2 zNxUBwd%(Wnmc1l)0z{WV`3XSB63C%Ffzi;J##%Rn}%#xC^G3fTfWu0H6PC zo8jANbjw1s95glvkJo_WBis-()%NrWO{Vv}pXEX6kyouxH+P{h$GP+| zNe1CQEXEK2yt*+4H_(Q{=taeg^6|?h)EIQR{Uc)X1Kebx!C4e)d~Xq9R@5n0i~Tjn z_D93WpDfan0K{12zQMGLdoTSIN@SdC`R6%sIrNV~!2|CtvVpxOUJD)=GP3!_|4Yz~ z82FrYwhnGUHdV{c20=VD|9O{mm8N!XQ?`YJH4W)oy{fy0;o4*KRr8^ENf zL3n`r&Zl2Xd|CE$$t1Azi?+fk^Q1xecA@jmf7n}8PMIxQKCY5JJFdC^;6LmILn*VR z%Ev9zXF)}7%>U;}?GnQi3w{21_vwgZehVYv3jt$nft3z~@R8?w_%rUSeCH!#mONac7 z-@bqDd-gbnLF&)fKEEZK2oOh#-uC>f$}U%w0AQbFv8)vPZ~b;j+yaQ%Ue^sS{_q(G zBb^cun_xrfsK1HI^Z@aQiSpp5t@2}@I1jD^f}qH(9Ch!H(BXXI2Z*`gD{Tqy|4*0x(eS{b^FLkszvJmIqvd}m{{M`q{~1v~jOR<=?EkA>8Y@fINOFJ< zyXke9B6dl9%o-14TK^UsM~H*ZZhx@x z(!~<)dLJqS3`SuX#_{MsZ^XTCz{Q((1cx^;vYBVhqp_0f5qT(A|K3&!T%jh_ru3a_FrW) znE8+ENa6J&9DF%gL-5{jI0|_B^3yzN?%E{)9Py-)dGMcil$Si_o!r&@& zQx&}$0!9z%gKL~|Fce`vdZ8bGi3l$)hV@ykvmko4Nbqo7!HUyOamR9ehIo>^EK|&OIFpH(C-K}p{njLm|Xs(YMf#3_fHP4`Hx<{1ze0D4#+*m zT^sewfiOa%Kl~cVhWVQR3LrMgq;!ObsY6=e2Q0vbNN{4UB{+W>KO&QD_CQ|Tj{I0gUEg@W4p}H`MH2L zebdFOlD}glC;)Fw{@z6E@k`_$3cPE+++q6QG<>WFi(XB4DbIB;U%Q_lb??_VEzsWj za=B*|-gwIM8u^~xW_#`jz8p)dm{yAV*WQy4Y~;V>g)h;%Z?mpl(;mK;{KCz-!(mkz z(?Qbtzk22g3x@FV+%Cr=_IW%uTmY7X@GO>1-P@_QU(b5d-}4)J1%?K~RVd*-0r$(D zP(@@rVZlZ+rc?Jv=%3%ZC(>9i5Lds$nCBVSc5NFMj<0Pgl*}t&?3E73R z+emEhcs6vuYN&f=&=@6?q3_h37^-pvg<5xC_4eBUKb z7{u(5^_L!`dSV5r-DTYdgMSwc&=XH^0V2M}hU{+y_;Ogi834{{hkA7i{rMIVJ!*hR zUosG9^hc%nHdX=1ON2Wr%JXlc7;s`J7^HIa{#lb-?`#3wi2`{_clI|i2tcTkJ4B2> z|Joftbm;OU03w1P$sqnF;!^>{NWVVLzgMhH4p4wI+p#-;6Ca5HMC!YVrhkq%pg$=mrnTEciH8?#9U;70N-WxhzHfbd(trHhIA%7cv!mZ z)zgU2`irshx8Efn@6I82y%DA1)-EXLgeV3>5~{D{SMpFW>yznN9k*7uH6IP}Dr|rD z@L!u&Pw84c+WK7oEIfP{wFpieY2lc};k|$&-jFXQnX~;~vxWD!v^{L$bx_K;#LxIK zyiYdW6=>HzvJ-D1uKmbp0~X>b?(v+rn-i$U=^=Wr)_Z0a1L+B9Z5a9txQxc01S31X zQ2{lCjZzRk^sYY%-+i`LYX~SLq$6bVx5k&aHDPXC$isy_!@uVu0L9UW@(f5C_)YCIOJKh>uQh{#FIPjkti2QQ&I( z|6i&t9$+fSPX!VFIoW_ABxwQFEN1gJRt!+vr2vqc*Yszu=f5ZU!$?3VaK9O<{}Nu= zxJIU;hD~Gi$u6oL?D1`H4KyocNDy~+?e^g~`e{E3P-X3>GskiYyE~Ju$xdfU-X)tR zd-J!>_s)nO%1^GAeNs2O{LCko81860Mku^Li-^$s*--#`yn(|~Bo0+}iy4e==R4S` zqM(Ph-1&d)U3)x}`yVgqC{7}kQbt6P(W^*dm#!H+>d5JOkABEzXPf{h#5Y~ei$psmdA?TN^_67ht z*~cB&5LR~r?zWPIlwA2JA#$cZ{fh_80o?W9!6h$1%l49*&q@dvhql!~);0!4zy2q( z<@&b*873ER`nde?W~q8*ft$hstCGEzaw!xuss7=hliOFh7am(36j-{YBd-0PRg_xR zqpdG*T-kluWXrhjY6;18az7MThvG;D4?W~m z;eo4yYJd;%Wf;=1jS2)JhCJ<$;xbh?gx5Y)d{&O+X&gGbutjykW$g&4=uX+y*?36# zfY37zLas?h2G&MMpQd1}+I$w)z|(vrs^_|JQ<<&twR5`SjRn!J0;)ma&HTs zs`S|22Pnu2G-iQ)cJ5v}1?cQhkzke2HW_VOyCg2D_T`wS2-SVC@d7f{#Hl6jTbzYA zTdSQ~<&%pw)k?E^ntB%eDnGW#ns(huh!(i**iVx+R;EvB(0oSEARz`q#r~~t&~V!# z8;o-C$h=erktIA$X&SAXv^~yv7@Hr5AAfFGdYE6@=jXuLe%7+KD_;>myXRV4or=j+ zmt&-f)8u!m{hRh_O1$)#p~)Lg5lojf+`I+jpW~HlRn537eiC;Q#k6bni%72iqp{;Q=$s%QOTQ2@p`Yp8m#yN8Xu zK8&}t+g4Q9Z2)1tj7kT7M3*TlFl8eBTkLUWp?^**esYl?kz0&DkZZVDreC{z$l%F2 z9UsZ=S%}N$I?t&x@L-&9w5>XzXTy=Es+-r1_EF8-Au-ZT%A&cR1L%g-WQ$Dg6$wO{ z(-d44ys?_?wAUj^IE1>kh3t^-`+b0u5#c6wlFH(izW{N_WI6nFfr3SUF8N;9F4Es- z?{dgDqk}DPe#=)WSKqJ??X~?#0|Z0gW0>mq=a&jQorbOa8#T21hJGBoTZ^S_zWK12 z+TvP19)+wKP2}m-H{J(Myw?KU&*ok(oCzTb4~0Y|x+@->WHRG?#_qzN8;AZeY8j@~ zyioP~b%UZKj2#OhIl76MIYroHhDXZY_*FsDYj?A@+_?0)7gWhk5xhtD&Qv5zXIVj+ zaaxThW^R?(+qg{oIbS%Qu$Xx{RdeH@pcgM*>1h^HI}pkfwY6|p*!(BEf&%l8ILc5| z_gc?)H5+c%kwc`C5&T{m*6GOHdB<+YLR@3hS1NnN+(4NUnqm^Yv+SPNM zO9{#`k+vhTh1W92_q5^36oP9t22`eB!>yJ%6`@x{k5R(E<#Mr1v+pd99`vIZT~sza zppF6&n#VwMjp3q=NE_UE(RTi(B)=yTD^&+utU9BN+hU|EfA$qjJfA~4FlSHXy?0ic zhLCKE$hwtHgi9H|llk8f=Uy7i^Euwgh8i3D7kjYty)3%ZXS!E-~`$AiN@hdv7E;y?dtcPcWY8_@Z=h%*` z_%_{&%5H+pRM~3c_Zx6Oo^YqcBZ0qUvz^Im$=kFfS#sSpDX_s`s&B8WBf;XQ1)}JM zpH8R%=wXUTlMxM>jlSQ)EFUQ|c!LMyKHxmVjH#%$!`h9oIUdn+SMgMrYl6D$kweO+ zvqM9{&pVN&Cqg(Evo}OSVN`Tpgg7NX7gZQI7L&(P)*=8DH=xMxkh8qnpUL z?Vb87q&kh)uVVS_rHY@n2Ft0;EOSJQjTGEe(`;*H$N4lS8^*luAL3VIi?kyx04#;mAG<+<4rP>A)`++5nw`_b@6!+L9+{= zanX|=gLpSd8_qL96B&6EeTeyIk=e&+V1?c;Xu{?<0rj6DoKz&ZbEPJncMW(ygV<+d zryM~?tBN7^-B4^tG8Sg$8vfAO4b{%{Y4?>4qaGj$y7h)i&UIRR-4lI7EeUuluReX8 z-xHp_Y)5Vs5T>TESm@Kz>Z1)fP`gO@8JVSP!UZ20XI-Pmw^U4o8Nkkk;+%+P(wD^( z@6~U&0Uw;REQw#lVfY1yUhnAn5=O#K6Q9XUu^<8rSfs z`RT6ly0!d=<)qY>3isB?8u#W<_6~2NSwn%(1Z!HunvQI}A-&koPX(R6-beFrmIWo_ z(M3wkthmp;?>Oi+rXq|QbWW$O$g9!ft-_kU_SKW~0#d!Q9P7{LjT*iJ$4H!;sKpw8 zuZ863_9+s7e+CB=G&d@On{99nOyl9e%_*3C!ug1Se(=Gc(1gcT03{ruP30vV3tFZC z#nl)}Q<;Sf4Z8n>!VO1XT9Uodup{AjQbe^^?n!SQRP~So0$mt#daMb`V|u5jtW=uy z!Wd*q^IqY>b>YY1gHTzW7mFQt5u3gXj2>Gxw$cmawuJ!AVCUsyfA5+$b#SQTs-P=c;NWq0w})`nLE4{ms{nYl^Z&y zv_4;n(+B{RM7^|cD~&-P2oo*b`_S1XBY=505nzJ5HGz+o`?+ez??6NQ*O-4JvrH!c zW6kW}udi=4rR8!^7!(wQ{;5sf67PUNPi^!x_*HxGeE|M#8HZ0TQz<+RDXDp75b8#& zf<~bS_K&oskZWy{mgyqRSV95pLE;UDWkpi?-C9>Q`UyHV7S2{0D;?R?A)V<_*nASU zf(gkv;270|z0dB!uH|K_=CJ^Dw9kl^hkxKAF%3BKp#1#`2mZU}{}P{C@v87+Hur1U S>Wo#u@65^bCte?O_~SpUDYZ`k literal 0 HcmV?d00001 diff --git a/docs/source/conceptual/speculative-decoding/main.py b/docs/source/conceptual/speculative-decoding/main.py new file mode 100644 index 00000000..6d421984 --- /dev/null +++ b/docs/source/conceptual/speculative-decoding/main.py @@ -0,0 +1,143 @@ +import numpy as np +import time +from tqdm import tqdm +from gpt2.utils import load_encoder_hparams_and_params +from gpt2.gpt import gpt2, softmax +from helper import get_sample, max_fn +import functools + + +def auto_reg_sampling(input_seq, model, N_future): + n = len(input_seq) + T = len(input_seq) + N_future + + with tqdm(total=N_future, desc="autoreg sampling") as pbar: + while n < T: + input_seq = np.append(input_seq, get_sample(model(input_seq)[-1])) + n += 1 + pbar.update(1) + return input_seq + + +def spec_sampling(input_seq, draft_model, target_model, N_future, k): + n = len(input_seq) + T = len(input_seq) + N_future + + with tqdm(total=N_future, desc="Spec Sampling") as pbar: + while n < T: + prev_n = n + # step1: autoreg generate from draft model and sample p + input_draft = input_seq + for _ in range(k): + p = draft_model(input_draft) # out logits + input_draft = np.append(input_draft, get_sample(p[-1])) + + # step2: input the whole seq of draft to target model + q = target_model(input_draft) + + # step3: Acceptance/ Rejection based on the p/q ratio + all_generated_tokens_accepted = True + for _ in range(k): + i = n - 1 + j = input_draft[i + 1] + + if np.random.random() < min(1, q[i][j] / p[i][j]): # accepted + input_seq = np.append(input_seq, j) + n += 1 + else: # rejected ---> resample from q-p + input_seq = np.append(input_seq, get_sample(max_fn(q[i] - p[i]))) + n += 1 + all_generated_tokens_accepted = False + break + + # step 4 + if all_generated_tokens_accepted: + input_seq = np.append(input_seq, get_sample(q[-1])) + n += 1 + + # for the bar + pbar.update(n - prev_n) + assert n == len(input_seq), f"{n} {len(input_seq)}" + return input_seq + + +def create_model_fn(params, hparams, temperature, eps=1e-10): + f = functools.partial(gpt2, **params, n_head=hparams["n_head"]) + + def model_fn(inputs): + logits = f(inputs) + logits = logits / (temperature + eps) # eps to avoid division by zero + probs = softmax(logits) + return probs + + return model_fn + + +def main( + prompt: str = "Quantization also improves latency and throughput but suffer from perf", + n_tokens_to_generate: int = 40, + draft_model_size: str = "124M", + target_model_size: str = "355M", + models_dir: str = "models", + K: int = 4, + temperature: float = 0.0, + seed: int = 123, +): + # seed numpy rng + np.random.seed(seed) + + # load encoder, hparams, and params from the released open-ai gpt-2 files + encoder, draft_hparams, draft_params = load_encoder_hparams_and_params( + draft_model_size, models_dir + ) + _, target_hparams, target_params = load_encoder_hparams_and_params( + target_model_size, models_dir + ) + draft_model = create_model_fn(draft_params, draft_hparams, temperature) + target_model = create_model_fn(target_params, target_hparams, temperature) + + # encode inputs + input_ids = encoder.encode(prompt) + + def run_sampling_fn(decode_fn, input_seq, **kwargs): + start = time.perf_counter() + output_ids = decode_fn(input_seq=input_seq, **kwargs) + text = encoder.decode(output_ids) + elapsed_time = time.perf_counter() - start + return text, elapsed_time + + # autoregressive sampling + autoregressive_text, autoregressive_time = run_sampling_fn( + auto_reg_sampling, + input_seq=input_ids, # Pass correct parameter + model=target_model, + N_future=n_tokens_to_generate, # Use N_future instead of N + ) + + # speculative sampling + speculative_text, speculative_time = run_sampling_fn( + spec_sampling, + input_seq=input_ids, # Pass correct parameter + target_model=target_model, + draft_model=draft_model, + N_future=n_tokens_to_generate, # Use N_future instead of N + k=K, + ) + + # print results + print() + print("Autoregressive Decoding:") + print("-" * 50) + print(f"Time = {autoregressive_time:.2f}s") + print(f"Text = {autoregressive_text}") + print() + print("Speculative Decoding:") + print("-" * 50) + print(f"Time = {speculative_time:.2f}s") + print(f"Text = {speculative_text}") + + +if __name__ == "__main__": + import fire + + fire.Fire(main) diff --git a/docs/source/conceptual/speculative-decoding/readme.md b/docs/source/conceptual/speculative-decoding/readme.md new file mode 100644 index 00000000..60520f89 --- /dev/null +++ b/docs/source/conceptual/speculative-decoding/readme.md @@ -0,0 +1,169 @@ +# ๐Ÿ”„๐Ÿ” Speculative Decoding +This project provides a simple implementation of the [Accelerating Large Language Model Decoding with Speculative Sampling](https://arxiv.org/abs/2302.01318) paper by Leviathan et al. The implementation uses pure NumPy for a basic GPT-2 model, demonstrating the concept of speculative decoding in a straightforward manner. + +Key features of this implementation: +- Uses NumPy for all computations, making it easy to understand and modify +- Implements speculative decoding for a GPT-2 model +- Compares performance between standard autoregressive sampling and speculative sampling +- Provides a clear example of how speculative decoding can accelerate language model inference + +This simple implementation serves as an educational tool to understand the core concepts of speculative decoding and its potential benefits in accelerating large language model inference. + +# Speculative Decoding in a nutshell +Speculative decoding is an innovative technique designed to accelerate the inference process of large language models. Here's a brief overview of how it works: + +1. Draft Model: A smaller, faster "draft" model generates a sequence of K tokens quickly. + +2. Target Model: The larger, more accurate "target" model processes the entire sequence (input + draft) in parallel. + +3. Verification: The target model's output is compared with the draft model's predictions. + +4. Accept or Reject: + - If the target model agrees with a draft token, it's accepted. + - If there's a disagreement, the draft is rejected, and the target model's prediction is used instead. + +5. Efficiency Gain: This approach allows the target model to process multiple tokens in a single forward pass, potentially reducing the number of expensive computations. + +The key advantage is that when the draft model's predictions are mostly correct, the process can be significantly faster than traditional autoregressive decoding. Even when the draft model makes mistakes, the performance doesn't degrade below that of standard autoregressive sampling. + +This method leverages the speed of smaller models and the accuracy of larger ones, offering a balance between inference speed and output quality. + + +# ๐Ÿš€ How to Use + +To run the speculative decoding implementation, use the following command: + +```bash +python main.py \ + --prompt "Quantization also improves latency and throughput but suffer from perf" \ + --n_tokens_to_generate 60 \ + --draft_model_size "124M" \ + --target_model_size "355M" \ + --K 4 \ + --temperature 0 # 0 for greedy sampling +``` +Sample Output: + +``` +Autoregressive Decoding +-------------------------------------------------- +Time = 112.19s +Text = Quantization also improves latency and throughput but suffer from perfomance issues. + +The problem is that the performance of the GPU is not the only thing that matters. The CPU is also important. The CPU is the main bottleneck in the GPU. The CPU is the main bottleneck in the GPU. + +The CPU is the main bottleneck in the GPU + +Speculative Decoding +-------------------------------------------------- +Time = 74.12s +Text = Quantization also improves latency and throughput but suffer from perfomance issues. + +The problem is that the performance of the GPU is not the only thing that matters. The CPU is also important. The CPU is the main bottleneck in the GPU. The CPU is the main bottleneck in the GPU. + +The CPU is the main bottleneck in the GPU. The CPU + +``` + + + +# ๐Ÿค”๐Ÿ’ญ Why this works? +Most of the work getting done is **NOT** about computation, but its actually about all those read/writes to access memory. +Bc whats happening is that the input lives on the memory and when you do any computation, it has to travel to the GPU/ to all the caches and registers to do the computation and then back to the memory. This is a very slow process. +![alt text](img/image.png) + +So each time we are doing round trips which is slow and very expensive. SO the idea is basically we gonna do a single trip to GPU and while that memory or at least a chunk of it is in the GPU, we are gonna do as much computation as possible and then we gonna load back the results to the memory. + +> "Now the clever idea is to use a small and cheap draft model to first generate a candidate sequence of K tokens - a 'draft'. Then we feed all of these together through the big model in a batch. This is almost as fast as feeding in just one token, per the above. Then we go from left to right over the logits predicted by the model and sample tokens. Any sample that agrees with the draft allows us to immediately skip forward to the next token. If there is a disagreement then we throw the draft away and eat the cost of doing some throwaway work (sampling the draft and the forward passing for all the later tokens). +> +> The reason this works in practice is that most of the time the draft tokens get accepted, because they are easy, so even a much smaller draft model gets them. As these easy tokens get accepted, we skip through those parts in leaps. The hard tokens where the big model disagrees 'fall back' to original speed, but actually a bit slower because of all the extra work." +> +> โ€” Andrej Karpathy + + + +# ๐Ÿงฎ๐Ÿ’กWhy this works mathematically? + +Speculative decoding's mathematical foundation is rooted in rejection sampling, a Monte Carlo method used to generate samples from a draft/smaller distribution when direct sampling from the target/larger distribution is difficult. + +## Mathematical Foundation: [Rejection Sampling](https://en.wikipedia.org/wiki/Rejection_sampling) + +Speculative decoding's mathematical foundation is rooted in rejection sampling, a Monte Carlo method used to generate samples from a target distribution when direct sampling is difficult. The process involves using a proposal distribution (the draft model) that's easier to sample from, then accepting or rejecting these samples based on comparison with the target distribution (the large model). The rejection sampling theorem guarantees that if we sample from the proposal distribution and accept samples with probability proportional to the ratio of target to proposal distributions, the accepted samples will follow the target distribution exactly. The reason of why this so magically works roots back to the bayes rule that we use to calculate the conditional probability of the next token given the previous context. + +## โŒ๐ŸŽฏ Rejection Sampling Theorem + +The theorem states that if we have a target distribution \( p \) and a proposal distribution \( q \), and we sample from \( q \) and accept samples with probability proportional to the ratio of \( p \) to \( q \), the accepted samples will follow the target distribution \( p \). + +Mathematically, this can be expressed as: + +1. Sample y from q(y) +2. Accept y with probability min(1, p(y) / (M * q(y))) + +Where: +- p(y) is the target distribution +- q(y) is the proposal distribution +- M is a constant such that M โ‰ฅ max(p(y) / q(y)) for all y + +If we follow this procedure, the accepted samples will be distributed according to p(y). + +## Question: What if we dont have access to the same family model for both draft and target model? + +Alternative methods like; + +- Medusa +- N-gram + + +### Medusa + + +Medusa is a [simple method](https://arxiv.org/abs/2401.10774) to create many tokens in a single pass using fine-tuned LM heads in addition to your existing models. + + +You can check a few existing fine-tunes for popular models: + +- [text-generation-inference/gemma-7b-it-medusa](https://huggingface.co/text-generation-inference/gemma-7b-it-medusa) +- [text-generation-inference/Mixtral-8x7B-Instruct-v0.1-medusa](https://huggingface.co/text-generation-inference/Mixtral-8x7B-Instruct-v0.1-medusa) +- [text-generation-inference/Mistral-7B-Instruct-v0.2-medusa](https://huggingface.co/text-generation-inference/Mistral-7B-Instruct-v0.2-medusa) + + +In order to create your own medusa heads for your own finetune, you should check own the original medusa repo. [../basic_tutorials/train_medusa.md](../basic_tutorials/train_medusa.md) + + +In order to use medusa models in TGI, simply point to a medusa enabled model, and everything will load automatically. + + +### N-gram + + +If you don't have a medusa model, or don't have the resource to fine-tune, you can try to use `n-gram`. +N-gram works by trying to find matching tokens in the previous sequence, and use those as speculation for generating new tokens. For example, if the tokens "np.mean" appear multiple times in the sequence, the model can speculate that the next continuation of the tokens "np." is probably also "mean". + +This is an extremely simple method, which works best for code, or highly repetitive text. This might not be beneficial, if the speculation misses too much. + + +In order to enable n-gram speculation simply use + +`--speculate 2` in your flags. [Details about the flag](https://huggingface.co/docs/text-generation-inference/basic_tutorials/launcher#speculate) + + +Please refer to [Speculation](https://huggingface.co/docs/text-generation-inference/conceptual/speculation) for more details. + + +# โšก๐Ÿš€ Summary of most common speed up techniques: +## ๐Ÿง ๐Ÿ’ป Faster Training +- Device: Move on to GPU +- Mix percisions +- Gradient Accumulation +- Distributed Training: + +## โšก๐Ÿค– Faster Inference +- Quantization +- Speculative Decoding (This repo ๐Ÿ’–) +- Pruning +- Caching + - inference-attention: KV cache + - in production: Prompt cache/ Exact cache/ Semantic cache +- Knowledge Distillation + + diff --git a/docs/source/conceptual/speculative-decoding/requirements.txt b/docs/source/conceptual/speculative-decoding/requirements.txt new file mode 100644 index 00000000..a65dc8f7 --- /dev/null +++ b/docs/source/conceptual/speculative-decoding/requirements.txt @@ -0,0 +1,6 @@ +fire==0.6.0 +numpy==2.1.1 +regex==2024.5.15 +Requests==2.32.3 +tensorflow==2.17.0 +tqdm==4.66.4