From 79ea7ff02fac0637db1a9178d5f8c27c814f02a4 Mon Sep 17 00:00:00 2001 From: rsnm2 Date: Sun, 20 Aug 2023 02:23:44 +0000 Subject: [PATCH] rewrote pipeline with simpler interface, working on adding to tgi now --- interaction.ipynb | 1691 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1691 insertions(+) create mode 100644 interaction.ipynb diff --git a/interaction.ipynb b/interaction.ipynb new file mode 100644 index 00000000..dd5fc582 --- /dev/null +++ b/interaction.ipynb @@ -0,0 +1,1691 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import deepsparse" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "os.environ[\"WAND_OPT_FLAGS\"] = \"default,~pyramids\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-19 21:47:03 deepsparse.transformers WARNING The neuralmagic fork of transformers may not be installed. It can be installed via `pip install nm_transformers`\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "fa1ef98341ae44e7a4787a05fad4c5e8", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Downloading (…)se/model.onnx.tar.gz: 0%| | 0.00/789M [00:00 127\n", + " engine.kv_cache._state = kv_cache_insert(engine.kv_cache._state, num_items=multitoken_engine.input_ids_length - engine.input_ids_length)\n", + "\n", + " # loop of singletoken engine for the rest\n", + " tokens_processed = engine.kv_cache.total_num_processed_tokens\n", + " while tokens_processed < len(tokens):\n", + " logits = decode(tokens[:tokens_processed + 1])\n", + " prompt_logits.append(logits)\n", + " tokens_processed += 1\n", + " \n", + " return prompt_logits" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": {}, + "outputs": [], + "source": [ + "def sample_token(logits):\n", + " return numpy.argmax(logits)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [], + "source": [ + "def kv_cache_slice(kv_cache, slice_idx):\n", + " for key in kv_cache:\n", + " kv_cache[key] = numpy.ascontiguousarray(kv_cache[key][:,:,slice_idx:,:])\n", + " return kv_cache\n", + "\n", + "def kv_cache_insert(kv_cache, num_items = 1, padding_value = 0):\n", + " indices = [0] * num_items\n", + " for key, value in kv_cache.items():\n", + " dtype = value.dtype\n", + " padding_value = numpy.array(padding_value, dtype=dtype)\n", + " kv_cache[key] = numpy.insert(value, indices, padding_value, axis=2)\n", + "\n", + " return kv_cache\n", + "\n", + "def call(eng, inputs):\n", + " inp = eng.add_kv_cache_to_input(inputs)\n", + " \n", + " logits, *kvs = eng.engine.run(inp, True)\n", + " new_kv_cache_state = {name: arr for name, arr in zip(cache_onnx_names, kvs)}\n", + "\n", + " eng.kv_cache.total_num_processed_tokens += eng.input_ids_length\n", + " eng.kv_cache._state = kv_cache_slice(new_kv_cache_state, eng.input_ids_length)\n", + "\n", + " return logits" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "128" + ] + }, + "execution_count": 37, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "engine.sequence_length" + ] + }, + { + "cell_type": "code", + "execution_count": 184, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318, 8639]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318, 8639, 416]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318, 8639, 416, 11271]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318, 8639, 416, 11271, 71]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318, 8639, 416, 11271, 71, 346]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318, 8639, 416, 11271, 71, 346, 26105]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318, 8639, 416, 11271, 71, 346, 26105, 14403]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318, 8639, 416, 11271, 71, 346, 26105, 14403, 7]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318, 8639, 416, 11271, 71, 346, 26105, 14403, 7, 17172]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318, 8639, 416, 11271, 71, 346, 26105, 14403, 7, 17172, 89]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318, 8639, 416, 11271, 71, 346, 26105, 14403, 7, 17172, 89, 1347]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318, 8639, 416, 11271, 71, 346, 26105, 14403, 7, 17172, 89, 1347, 62]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318, 8639, 416, 11271, 71, 346, 26105, 14403, 7, 17172, 89, 1347, 62, 25816]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318, 8639, 416, 11271, 71, 346, 26105, 14403, 7, 17172, 89, 1347, 62, 25816, 8]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318, 8639, 416, 11271, 71, 346, 26105, 14403, 7, 17172, 89, 1347, 62, 25816, 8, 198]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318, 8639, 416, 11271, 71, 346, 26105, 14403, 7, 17172, 89, 1347, 62, 25816, 8, 198, 50256]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318, 8639, 416, 11271, 71, 346, 26105, 14403, 7, 17172, 89, 1347, 62, 25816, 8, 198, 50256, 2]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318, 8639, 416, 11271, 71, 346, 26105, 14403, 7, 17172, 89, 1347, 62, 25816, 8, 198, 50256, 2, 48443]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318, 8639, 416, 11271, 71, 346, 26105, 14403, 7, 17172, 89, 1347, 62, 25816, 8, 198, 50256, 2, 48443, 14629]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318, 8639, 416, 11271, 71, 346, 26105, 14403, 7, 17172, 89, 1347, 62, 25816, 8, 198, 50256, 2, 48443, 14629, 14]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318, 8639, 416, 11271, 71, 346, 26105, 14403, 7, 17172, 89, 1347, 62, 25816, 8, 198, 50256, 2, 48443, 14629, 14, 8800]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318, 8639, 416, 11271, 71, 346, 26105, 14403, 7, 17172, 89, 1347, 62, 25816, 8, 198, 50256, 2, 48443, 14629, 14, 8800, 14]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318, 8639, 416, 11271, 71, 346, 26105, 14403, 7, 17172, 89, 1347, 62, 25816, 8, 198, 50256, 2, 48443, 14629, 14, 8800, 14, 24330]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318, 8639, 416, 11271, 71, 346, 26105, 14403, 7, 17172, 89, 1347, 62, 25816, 8, 198, 50256, 2, 48443, 14629, 14, 8800, 14, 24330, 21015]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318, 8639, 416, 11271, 71, 346, 26105, 14403, 7, 17172, 89, 1347, 62, 25816, 8, 198, 50256, 2, 48443, 14629, 14, 8800, 14, 24330, 21015, 198]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318, 8639, 416, 11271, 71, 346, 26105, 14403, 7, 17172, 89, 1347, 62, 25816, 8, 198, 50256, 2, 48443, 14629, 14, 8800, 14, 24330, 21015, 198, 2]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318, 8639, 416, 11271, 71, 346, 26105, 14403, 7, 17172, 89, 1347, 62, 25816, 8, 198, 50256, 2, 48443, 14629, 14, 8800, 14, 24330, 21015, 198, 2, 532]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318, 8639, 416, 11271, 71, 346, 26105, 14403, 7, 17172, 89, 1347, 62, 25816, 8, 198, 50256, 2, 48443, 14629, 14, 8800, 14, 24330, 21015, 198, 2, 532, 9]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318, 8639, 416, 11271, 71, 346, 26105, 14403, 7, 17172, 89, 1347, 62, 25816, 8, 198, 50256, 2, 48443, 14629, 14, 8800, 14, 24330, 21015, 198, 2, 532, 9, 12]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318, 8639, 416, 11271, 71, 346, 26105, 14403, 7, 17172, 89, 1347, 62, 25816, 8, 198, 50256, 2, 48443, 14629, 14, 8800, 14, 24330, 21015, 198, 2, 532, 9, 12, 19617]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318, 8639, 416, 11271, 71, 346, 26105, 14403, 7, 17172, 89, 1347, 62, 25816, 8, 198, 50256, 2, 48443, 14629, 14, 8800, 14, 24330, 21015, 198, 2, 532, 9, 12, 19617, 25]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318, 8639, 416, 11271, 71, 346, 26105, 14403, 7, 17172, 89, 1347, 62, 25816, 8, 198, 50256, 2, 48443, 14629, 14, 8800, 14, 24330, 21015, 198, 2, 532, 9, 12, 19617, 25, 3384]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318, 8639, 416, 11271, 71, 346, 26105, 14403, 7, 17172, 89, 1347, 62, 25816, 8, 198, 50256, 2, 48443, 14629, 14, 8800, 14, 24330, 21015, 198, 2, 532, 9, 12, 19617, 25, 3384, 69]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318, 8639, 416, 11271, 71, 346, 26105, 14403, 7, 17172, 89, 1347, 62, 25816, 8, 198, 50256, 2, 48443, 14629, 14, 8800, 14, 24330, 21015, 198, 2, 532, 9, 12, 19617, 25, 3384, 69, 12]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318, 8639, 416, 11271, 71, 346, 26105, 14403, 7, 17172, 89, 1347, 62, 25816, 8, 198, 50256, 2, 48443, 14629, 14, 8800, 14, 24330, 21015, 198, 2, 532, 9, 12, 19617, 25, 3384, 69, 12, 23]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318, 8639, 416, 11271, 71, 346, 26105, 14403, 7, 17172, 89, 1347, 62, 25816, 8, 198, 50256, 2, 48443, 14629, 14, 8800, 14, 24330, 21015, 198, 2, 532, 9, 12, 19617, 25, 3384, 69, 12, 23, 532]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318, 8639, 416, 11271, 71, 346, 26105, 14403, 7, 17172, 89, 1347, 62, 25816, 8, 198, 50256, 2, 48443, 14629, 14, 8800, 14, 24330, 21015, 198, 2, 532, 9, 12, 19617, 25, 3384, 69, 12, 23, 532, 9]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318, 8639, 416, 11271, 71, 346, 26105, 14403, 7, 17172, 89, 1347, 62, 25816, 8, 198, 50256, 2, 48443, 14629, 14, 8800, 14, 24330, 21015, 198, 2, 532, 9, 12, 19617, 25, 3384, 69, 12, 23, 532, 9, 12]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318, 8639, 416, 11271, 71, 346, 26105, 14403, 7, 17172, 89, 1347, 62, 25816, 8, 198, 50256, 2, 48443, 14629, 14, 8800, 14, 24330, 21015, 198, 2, 532, 9, 12, 19617, 25, 3384, 69, 12, 23, 532, 9, 12, 198]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318, 8639, 416, 11271, 71, 346, 26105, 14403, 7, 17172, 89, 1347, 62, 25816, 8, 198, 50256, 2, 48443, 14629, 14, 8800, 14, 24330, 21015, 198, 2, 532, 9, 12, 19617, 25, 3384, 69, 12, 23, 532, 9, 12, 198, 198]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318, 8639, 416, 11271, 71, 346, 26105, 14403, 7, 17172, 89, 1347, 62, 25816, 8, 198, 50256, 2, 48443, 14629, 14, 8800, 14, 24330, 21015, 198, 2, 532, 9, 12, 19617, 25, 3384, 69, 12, 23, 532, 9, 12, 198, 198, 37811]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318, 8639, 416, 11271, 71, 346, 26105, 14403, 7, 17172, 89, 1347, 62, 25816, 8, 198, 50256, 2, 48443, 14629, 14, 8800, 14, 24330, 21015, 198, 2, 532, 9, 12, 19617, 25, 3384, 69, 12, 23, 532, 9, 12, 198, 198, 37811, 198]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318, 8639, 416, 11271, 71, 346, 26105, 14403, 7, 17172, 89, 1347, 62, 25816, 8, 198, 50256, 2, 48443, 14629, 14, 8800, 14, 24330, 21015, 198, 2, 532, 9, 12, 19617, 25, 3384, 69, 12, 23, 532, 9, 12, 198, 198, 37811, 198, 50284]\n", + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, 198, 50284, 361, 299, 6624, 657, 25, 198, 50280, 7783, 657, 198, 50284, 417, 361, 299, 6624, 352, 25, 198, 50280, 7783, 352, 198, 50284, 17772, 25, 198, 50280, 7783, 12900, 7, 77, 12, 16, 8, 1343, 12900, 7, 77, 12, 17, 8, 198, 198, 2, 4889, 262, 2163, 13, 198, 4798, 7, 69, 571, 7, 20, 4008, 198, 198, 2, 770, 2438, 318, 8639, 416, 11271, 71, 346, 26105, 14403, 7, 17172, 89, 1347, 62, 25816, 8, 198, 50256, 2, 48443, 14629, 14, 8800, 14, 24330, 21015, 198, 2, 532, 9, 12, 19617, 25, 3384, 69, 12, 23, 532, 9, 12, 198, 198, 37811, 198, 50284, 27730]\n" + ] + } + ], + "source": [ + "tokens = engine_input[0][engine_input[1].nonzero()].tolist()\n", + "pipeline._reset_engines_cache()\n", + "\n", + "print(tokens)\n", + "logits = prefill(tokens)\n", + "tokens.append(sample_token(logits[-1][0,-1,:])) # assume always batch = 1, last token of last logit in array\n", + "\n", + "# first token from prefill was generated\n", + "while len(tokens) < engine.sequence_length:\n", + " print(tokens)\n", + " logits = decode(tokens)\n", + " tokens.append(sample_token(logits[0,-1,:])) # assume always batch = 1, last token of last logit in array\n", + "\n", + "# print(engine.kv_cache._state[\"past_key_values.0.key\"][0,0,-INDEX:,0])\n", + "# print(engine.kv_cache._state[\"past_key_values.0.value\"][0,0,-INDEX:,0])\n", + "# print(engine.kv_cache._state[\"past_key_values.19.key\"][0,0,-INDEX:,0])\n", + "# print(engine.kv_cache._state[\"past_key_values.19.value\"][0,0,-INDEX:,0])" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Finish the following function for computing a fibonacci sequence: \n", + "\n", + " fib(n):\n", + "\n", + " if n == 0:\n", + " return 0\n", + " elif n == 1:\n", + " return 1\n", + " else:\n", + " return fib(n-1) + fib(n-2)\n", + "\n", + "# Call the function.\n", + "print(fib(5))\n", + "\n", + "# This code is contributed by Nikhil Kumar Singh(nickzuck_007)\n", + "<|endoftext|>#!/usr/bin/env python\n", + "# -*- coding: utf-8 -*-\n", + "\n", + "\"\"\"\n", + " Examples for\n" + ] + } + ], + "source": [ + "print(pipeline.tokenizer.decode(tokens))" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[-2.16584 -1.4835675 -0.82798475 2.728714 2.230249 0.136684\n", + " -2.4744277 -3.7903032 -0.44804883 1.8597361 3.2892575 1.1238453\n", + " -0.535056 -2.4058022 -2.6181865 0.82309175 3.7468169 0.9127281\n", + " -0.08818069 -3.5193567 -2.554974 0.42606103 2.8396277 3.6084752\n", + " 0.720097 -3.140173 -2.3983316 -1.1198903 1.4021769 2.4038355\n", + " 1.416564 -1.1770982 ]\n", + "[-0.11449916 0.47542867 0.03680322 -0.4064121 0.34018266 0.11729242\n", + " -0.29119202 1.6026565 -0.60162723 1.6026565 0.03134436 0.005808\n", + " 0.03680322 0.34018266 1.0944448 0.89051616 1.9490317 -0.8315846\n", + " 0.8142212 -0.22642836 0.36906892 1.9490317 -0.15412031 1.0944448\n", + " 0.89051616 1.9490317 0.03680322 0.03680322 0.55909103 0.03680322\n", + " 0.76835376 0.07582108]\n", + "[-1.4705226 2.3718867 1.5622201 0.20804703 -0.930273 -5.223105\n", + " -2.31877 3.5253658 3.8794327 3.3048825 -2.4029026 -2.4765668\n", + " -0.68623084 1.3053839 6.9972997 4.6631894 -0.957654 -4.965276\n", + " -5.222634 0.77317643 5.6226482 6.351179 1.0147996 -5.322752\n", + " -5.885022 -1.1356002 0.9603227 2.44311 2.3220952 -1.8733013\n", + " -5.0550013 -2.9907336 ]\n", + "[-0.16411448 -0.05435281 -0.22059102 0.09352674 -0.05225876 -0.22478615\n", + " 0.4103162 -0.1921539 0.11564742 -0.38469723 -0.01235063 0.29627988\n", + " -0.06217921 0.3747058 0.1442022 0.31203395 0.669638 0.40900382\n", + " 0.34937513 0.07317603 0.49499115 -0.26419586 0.14836667 0.41960722\n", + " 0.53298324 0.6752395 0.5533317 0.20957318 0.25364277 0.08110742\n", + " -0.19118905 0.845217 ]\n" + ] + } + ], + "source": [ + "pipeline._reset_engines_cache()\n", + "\n", + "onnx_input_names = (\n", + " pipeline.multitoken_engine.onnx_input_names_no_cache\n", + " if pipeline.multitoken_engine\n", + " else pipeline.engine.onnx_input_names_no_cache\n", + ")\n", + "engine_input = pipeline.tokens_to_engine_input(input_tokens, onnx_input_names)\n", + "tokens_theirs, logits_theirs = pipeline.prompt_inference(engine_input)\n", + "\n", + "while len(tokens_theirs) < pipeline.sequence_length:\n", + " token, logits = pipeline.autoregressive_inference(tokens_theirs)\n", + " tokens_theirs.append(token)\n", + " \n", + "print(engine.kv_cache._state[\"past_key_values.0.key\"][0,0,-INDEX:,0])\n", + "print(engine.kv_cache._state[\"past_key_values.0.value\"][0,0,-INDEX:,0])\n", + "print(engine.kv_cache._state[\"past_key_values.19.key\"][0,0,-INDEX:,0])\n", + "print(engine.kv_cache._state[\"past_key_values.19.value\"][0,0,-INDEX:,0])" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Finish the following function for computing a fibonacci sequence: \n", + "\n", + " fib(n):\n", + "\n", + " if n == 0:\n", + " return 0\n", + " elif n == 1:\n", + " return 1\n", + " else:\n", + " return fib(n-1) + fib(n-2)\n", + "\n", + "# Call the function.\n", + "print(fib(5))\n", + "\n", + "# This code is contributed by Nikhil Kumar Singh(nickzuck_007)\n", + "<|endoftext|>#!/usr/bin/env python\n", + "# -*- coding: utf-8 -*-\n", + "\n", + "\"\"\"\n", + " Examples for\n" + ] + } + ], + "source": [ + "print(pipeline.tokenizer.decode(tokens_theirs))" + ] + }, + { + "cell_type": "code", + "execution_count": 350, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(1, 16, 127, 64)" + ] + }, + "execution_count": 350, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "engine.kv_cache._state[\"past_key_values.0.key\"].shape" + ] + }, + { + "cell_type": "code", + "execution_count": 321, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(2, 4, 124, 16)\n" + ] + } + ], + "source": [ + "arr = numpy.ones((2,4,124,16), dtype=numpy.uint8)\n", + "print(arr.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 334, + "metadata": {}, + "outputs": [], + "source": [ + "indices = [0] * 3\n", + "updated = numpy.insert(arr, indices, 0, axis=2)" + ] + }, + { + "cell_type": "code", + "execution_count": 335, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n", + "True\n" + ] + } + ], + "source": [ + "print(arr.data.contiguous)\n", + "print(updated.data.contiguous)" + ] + }, + { + "cell_type": "code", + "execution_count": 302, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n" + ] + } + ], + "source": [ + "print(engine.kv_cache)\n", + "print(multitoken_engine.kv_cache)" + ] + }, + { + "cell_type": "code", + "execution_count": 305, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "128\n", + "1\n", + "128\n", + "4\n", + "124\n", + "124\n" + ] + } + ], + "source": [ + "print(engine.sequence_length)\n", + "print(engine.input_ids_length)\n", + "\n", + "print(multitoken_engine.sequence_length)\n", + "print(multitoken_engine.input_ids_length)\n", + "\n", + "print(engine.kv_cache.capacity)\n", + "print(multitoken_engine.kv_cache.capacity)" + ] + }, + { + "cell_type": "code", + "execution_count": 297, + "metadata": {}, + "outputs": [], + "source": [ + "engine.transfer_cache_state(cache=multitoken_engine.kv_cache)" + ] + }, + { + "cell_type": "code", + "execution_count": 300, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "127\n" + ] + } + ], + "source": [ + "print(multitoken_engine.kv_cache.capacity)" + ] + }, + { + "cell_type": "code", + "execution_count": 299, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "127\n" + ] + } + ], + "source": [ + "print(engine.kv_cache.capacity)" + ] + }, + { + "cell_type": "code", + "execution_count": 128, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0\n", + "input_ids:\n", + "[[16594 257 2163 284]]\n", + "(1, 4)\n", + "attention_mask:\n", + "[[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n", + " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n", + " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n", + " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1]]\n", + "(1, 128)\n", + "positions:\n", + "[[0 1 2 3]]\n", + "(1, 4)\n", + "causal_mask:\n", + "[[[[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n", + " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n", + " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n", + " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0]\n", + " [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n", + " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n", + " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n", + " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 0 0]\n", + " [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n", + " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n", + " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n", + " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 0]\n", + " [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n", + " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n", + " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n", + " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1]]]]\n", + "(1, 1, 4, 128)\n", + "\n", + "\n", + "\n", + "4\n", + "input_ids:\n", + "[[24061 257 12900 261]]\n", + "(1, 4)\n", + "attention_mask:\n", + "[[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n", + " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n", + " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n", + " 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1]]\n", + "(1, 128)\n", + "positions:\n", + "[[4 5 6 7]]\n", + "(1, 4)\n", + "causal_mask:\n", + "[[[[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n", + " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n", + " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n", + " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 0 0 0]\n", + " [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n", + " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n", + " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n", + " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 0 0]\n", + " [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n", + " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n", + " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n", + " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 0]\n", + " [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n", + " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n", + " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n", + " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1]]]]\n", + "(1, 1, 4, 128)\n", + "\n", + "\n", + "\n" + ] + } + ], + "source": [ + "pipeline._reset_engines_cache()\n", + "\n", + "for engine_inputs in engine_inputs_for_prefill(tokens):\n", + " multitoken_engine(engine_inputs)\n", + " print(\"\\n\\n\")" + ] + }, + { + "cell_type": "code", + "execution_count": 272, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using pad_token, but it is not set yet.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Finish the following function for computing a fibonacci sequence: \n", + "\n", + " fib(n):\n", + "\n", + " if n == 0:\n", + " return 0\n", + " elif n == 1:\n", + " return 1\n", + " else:\n", + " return fib(n-1) + fib(n-2)\n", + "\n", + "# Call the function.\n", + "print(fib(5))\n", + "\n", + "# This code is contributed by Nikhil Kumar Singh(nickzuck_007)\n", + "<|endoftext|>\n" + ] + } + ], + "source": [ + "from typing import Optional, List, Dict\n", + "from deepsparse import Context\n", + "from deepsparse.pipeline import DEEPSPARSE_ENGINE, create_engine\n", + "from deepsparse.transformers.utils.helpers import overwrite_onnx_model_inputs\n", + "from transformers import AutoTokenizer\n", + "\n", + "class DecoderEngine:\n", + " def __init__ (\n", + " self,\n", + " onnx_file_path: str, \n", + " sequence_length: int = 1024,\n", + " input_ids_length: int = 1,\n", + " engine_context: Optional[Context] = None,\n", + " ):\n", + "\n", + " onnx_file_path, _, data_type = overwrite_onnx_model_inputs(\n", + " onnx_file_path=onnx_file_path,\n", + " batch_size=1,\n", + " sequence_length=sequence_length,\n", + " input_ids_length=input_ids_length,\n", + " )\n", + "\n", + " self.past_key_value_dtype = data_type\n", + " self.engine = create_engine(\n", + " onnx_file_path=onnx_file_path,\n", + " engine_type=DEEPSPARSE_ENGINE,\n", + " engine_args={},\n", + " context=engine_context,\n", + " )\n", + " print(self.engine)\n", + "\n", + " self.onnx_inputs = self.engine.input_names\n", + " \n", + " self.past_onnx_inputs = [\n", + " name for name in self.engine.input_names\n", + " if name.startswith(\"past_key_values\")\n", + " ]\n", + "\n", + " self.non_past_onnx_inputs = [\n", + " name for name in self.engine.input_names\n", + " if not name.startswith(\"past_key_values\")\n", + " ]\n", + " \n", + " def __call__(\n", + " self,\n", + " inputs: Dict[str, numpy.ndarray],\n", + " past_key_values: Dict[str, numpy.ndarray],\n", + " val_inp: bool = True\n", + " ):\n", + " # format input\n", + " inp = [past_key_values[name] if name.startswith(\"past_key_values\") \n", + " else inputs[name] for name in self.engine.input_names]\n", + "\n", + " # run inference\n", + " logits, *kvs = self.engine.run(inp, True)\n", + " past_key_values = {name: arr for name, arr in zip(self.past_onnx_inputs, kvs)}\n", + " \n", + " return logits, past_key_values\n", + "\n", + "\n", + "class Model:\n", + " def __init__(\n", + " self,\n", + " onnx_file_path: str,\n", + " sequence_length: int = 1024,\n", + " multi_token_length: int = 16,\n", + " engine_context: Optional[Context] = None,\n", + " singletoken_engine = None,\n", + " multitoken_engine = None,\n", + " ):\n", + " self.sequence_length = sequence_length\n", + " self.multi_token_length = multi_token_length\n", + "\n", + " if singletoken_engine is not None and multitoken_engine is not None:\n", + " self.singletoken_engine = singletoken_engine\n", + " self.multitoken_engine = multitoken_engine\n", + " else:\n", + " self.singletoken_engine = DecoderEngine(\n", + " onnx_file_path=onnx_file_path,\n", + " engine_context=engine_context,\n", + " sequence_length=sequence_length,\n", + " input_ids_length=1,\n", + " )\n", + " \n", + " self.multitoken_engine = DecoderEngine(\n", + " onnx_file_path=onnx_file_path,\n", + " engine_context=engine_context,\n", + " sequence_length=sequence_length,\n", + " input_ids_length=self.multi_token_length,\n", + " )\n", + "\n", + " assert self.multitoken_engine.past_key_value_dtype == self.singletoken_engine.past_key_value_dtype\n", + " self.past_key_value_dtype = self.multitoken_engine.past_key_value_dtype\n", + " \n", + " assert len(self.singletoken_engine.non_past_onnx_inputs) == 4\n", + " assert \"input_ids\" in self.singletoken_engine.non_past_onnx_inputs\n", + " assert \"attention_mask\" in self.singletoken_engine.non_past_onnx_inputs\n", + " assert \"causal_mask\" in self.singletoken_engine.non_past_onnx_inputs\n", + " assert \"positions\" in self.singletoken_engine.non_past_onnx_inputs\n", + "\n", + " # create empty kv caches with the proper sizes based on onnx graph\n", + " def init_past_key_values(self):\n", + " past_key_values = {}\n", + " for idx, name in enumerate(self.multitoken_engine.onnx_inputs):\n", + " if name.startswith(\"past_key_values\"):\n", + " shape = self.multitoken_engine.engine.input_shapes[idx]\n", + " past_key_values[name] = numpy.zeros(shape, dtype=self.past_key_value_dtype)\n", + "\n", + " return past_key_values\n", + "\n", + " # insert into every K,V matrix in the list\n", + " # BAD [SLOW] --- A copy of arr with values inserted. Note that insert does not occur in-place: a new array is returned. If axis is None, out is a flattened array.\n", + " def insert_past_key_values(self, past_key_values, num_items=1, padding_value=0):\n", + " for name in past_key_values:\n", + " padding_value = numpy.array(padding_value, dtype=self.past_key_value_dtype)\n", + " past_key_values[name] = numpy.insert(past_key_values[name], [0]*num_items, padding_value, axis=2)\n", + " return past_key_values\n", + "\n", + " # slice every K,V matrix in the list\n", + " # BAD [SLOW] --- calls .ascontinugousarray\n", + " def slice_past_key_values(self, past_key_values, slice_idx):\n", + " for name in past_key_values:\n", + " past_key_values[name] = numpy.ascontiguousarray(past_key_values[name][:,:,slice_idx:,:])\n", + " return past_key_values\n", + " \n", + " # slice input tokens into groups, make inputs dict\n", + " def engine_inputs_for_prefill(self, tokens):\n", + " num_batches = len(tokens) // self.multi_token_length\n", + " token_batches = [tokens[i * self.multi_token_length : (i+1) * self.multi_token_length] for i in range(0, num_batches)]\n", + "\n", + " num_processed_tokens = 0\n", + " for idx, token_batch in enumerate(token_batches):\n", + " engine_inputs = {}\n", + " engine_inputs[\"input_ids\"] = numpy.array([token_batch])\n", + "\n", + " # make attention mask from the right\n", + " engine_inputs[\"attention_mask\"] = numpy.zeros((1, self.sequence_length), dtype=numpy.int64)\n", + " engine_inputs[\"attention_mask\"][:, -(self.multi_token_length + num_processed_tokens):] = 1\n", + " \n", + " # make positions (building from the right)\n", + " assert self.multi_token_length > 1\n", + " engine_inputs[\"positions\"] = numpy.arange(\n", + " num_processed_tokens, num_processed_tokens + self.multi_token_length\n", + " ).reshape(1, -1).astype(numpy.int64)\n", + "\n", + " # make causal mask (building from the right)\n", + " engine_inputs[\"causal_mask\"] = create_causal_mask(\n", + " input_ids=engine_inputs[\"input_ids\"], \n", + " attention_mask=engine_inputs[\"attention_mask\"]\n", + " )\n", + "\n", + "\n", + " yield engine_inputs\n", + "\n", + " def engine_inputs_for_decode(self, tokens):\n", + " assert(len(tokens) < self.sequence_length)\n", + " \n", + " engine_inputs = {}\n", + " engine_inputs[\"input_ids\"] = numpy.array([[tokens[-1]]])\n", + " engine_inputs[\"attention_mask\"] = numpy.zeros((1, self.sequence_length), dtype=numpy.int64)\n", + " engine_inputs[\"attention_mask\"][:, -len(tokens):] = 1\n", + " \n", + " engine_inputs[\"causal_mask\"] = create_causal_mask(\n", + " engine_inputs[\"input_ids\"], \n", + " engine_inputs[\"attention_mask\"]\n", + " )\n", + " engine_inputs[\"positions\"] = numpy.array([[len(tokens) - 1]], dtype=numpy.int64)\n", + " \n", + " return engine_inputs\n", + " \n", + " # run prefill inference\n", + " def prefill(self, tokens):\n", + " assert len(tokens) < self.sequence_length - 1\n", + " \n", + " # initialize state\n", + " past_key_values = self.init_past_key_values()\n", + " tokens_processed = 0\n", + " \n", + " # loop over multitoken engine\n", + " for inputs in self.engine_inputs_for_prefill(tokens): \n", + " logits, past_key_values = self.multitoken_engine(inputs, past_key_values)\n", + " tokens_processed += self.multi_token_length\n", + " \n", + " # (this is BAD - calls np.ascontiguous) - cleanup past_kv state \n", + " past_key_values = self.slice_past_key_values(past_key_values, self.multi_token_length)\n", + " \n", + " # (this is BAD - returns a copy) - expand kv cache for single token engine \n", + " past_key_values = self.insert_past_key_values(past_key_values, num_items=(self.multi_token_length-1))\n", + "\n", + " # loop of singletoken engine for anything left over\n", + " while tokens_processed < len(tokens):\n", + " logits, past_key_values = self.decode(\n", + " tokens=tokens[:tokens_processed+1],\n", + " past_key_values=past_key_values\n", + " )\n", + " tokens_processed += 1\n", + "\n", + " assert logits.shape[0] == 1 # assert batch 1 right now\n", + " return logits[:,:,:], past_key_values\n", + " \n", + " # run decode inference\n", + " def decode(self, tokens, past_key_values): \n", + " engine_inputs = self.engine_inputs_for_decode(tokens)\n", + "\n", + " logits, past_key_values = self.singletoken_engine(\n", + " inputs=engine_inputs,\n", + " past_key_values=past_key_values\n", + " )\n", + "\n", + " # cleanup state (this is BAD - calls np.ascontiguous)\n", + " past_key_values = self.slice_past_key_values(past_key_values, 1)\n", + "\n", + " assert logits.shape[0] == 1 # assert batch 1 right now\n", + " assert logits.shape[1] == 1 # assert only one element\n", + " return logits[:,:,:], past_key_values\n", + "\n", + "def sample_token(logits):\n", + " assert(logits.shape[0] == 1)\n", + " return numpy.argmax(logits[0,-1,:])\n", + "\n", + "model = Model(\n", + " onnx_file_path=onnx_path,\n", + " sequence_length=128,\n", + " multi_token_length=16,\n", + " singletoken_engine=model.singletoken_engine,\n", + " multitoken_engine=model.multitoken_engine\n", + ")\n", + "\n", + "tokenizer = AutoTokenizer.from_pretrained(model_path)\n", + "tokenizer.padding_side = \"left\"\n", + "if not tokenizer.pad_token:\n", + " tokenizer.pad_token = tokenizer.eos_token\n", + "\n", + "def generate(model, tokenizer, text):\n", + " input_tokens = tokenizer(text, return_tensors=\"np\", max_length=model.sequence_length, padding=\"longest\", truncation=False,)\n", + " tokens = input_tokens[\"input_ids\"][input_tokens[\"attention_mask\"].nonzero()].tolist()\n", + "\n", + " # prefill\n", + " logits, past_key_values = model.prefill(tokens)\n", + " tokens.append(sample_token(logits))\n", + "\n", + " # run decode\n", + " while len(tokens) < model.sequence_length and tokens[-1] != tokenizer.eos_token_id:\n", + " logits, past_key_values = model.decode(tokens, past_key_values)\n", + " tokens.append(sample_token(logits))\n", + " \n", + " return tokens\n", + " \n", + "tokens = generate(model, tokenizer, sequence)\n", + "print(tokenizer.decode(tokens))" + ] + }, + { + "cell_type": "code", + "execution_count": 263, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "print(type(tokenizer.eos_token_id))" + ] + }, + { + "cell_type": "code", + "execution_count": 266, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261, 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599, 198, array([0, 0, 0, ..., 0, 0, 0])]\n" + ] + } + ], + "source": [ + "print(tokens)" + ] + }, + { + "cell_type": "code", + "execution_count": 220, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(1, 51200)" + ] + }, + "execution_count": 220, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "logits.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 205, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-20 01:52:24 deepsparse.transformers.utils.helpers INFO Overwriting in-place the input shapes of the transformer model at /home/robertgshaw/.cache/sparsezoo/neuralmagic/codegen_mono-350m-bigpython_bigquery_thepile-base/model.onnx/model.onnx\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "deepsparse.engine.Engine:\n", + "\tonnx_file_path: /home/robertgshaw/.cache/sparsezoo/neuralmagic/codegen_mono-350m-bigpython_bigquery_thepile-base/model.onnx/model.onnx\n", + "\tbatch_size: 1\n", + "\tnum_cores: 8\n", + "\tnum_streams: 1\n", + "\tscheduler: Scheduler.default\n", + "\tfraction_of_supported_ops: 1.0\n", + "\tcpu_avx_type: avx2\n", + "\tcpu_vnni: False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-20 01:52:54 deepsparse.transformers.utils.helpers INFO Overwriting in-place the input shapes of the transformer model at /home/robertgshaw/.cache/sparsezoo/neuralmagic/codegen_mono-350m-bigpython_bigquery_thepile-base/model.onnx/model.onnx\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "deepsparse.engine.Engine:\n", + "\tonnx_file_path: /home/robertgshaw/.cache/sparsezoo/neuralmagic/codegen_mono-350m-bigpython_bigquery_thepile-base/model.onnx/model.onnx\n", + "\tbatch_size: 1\n", + "\tnum_cores: 8\n", + "\tnum_streams: 1\n", + "\tscheduler: Scheduler.default\n", + "\tfraction_of_supported_ops: 1.0\n", + "\tcpu_avx_type: avx2\n", + "\tcpu_vnni: False\n" + ] + } + ], + "source": [ + "onnx_path = \"/home/robertgshaw/.cache/sparsezoo/neuralmagic/codegen_mono-350m-bigpython_bigquery_thepile-base/model.onnx/model.onnx\"\n", + "model_path = \"/home/robertgshaw/.cache/sparsezoo/neuralmagic/codegen_mono-350m-bigpython_bigquery_thepile-base/deployment\"\n", + "\n", + "model = Model(\n", + " onnx_file_path=onnx_path,\n", + " sequence_length=128,\n", + " multi_token_length=16\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 196, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using pad_token, but it is not set yet.\n" + ] + } + ], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 197, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "18" + ] + }, + "execution_count": 197, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(tokens)" + ] + }, + { + "cell_type": "code", + "execution_count": 206, + "metadata": {}, + "outputs": [ + { + "ename": "RuntimeError", + "evalue": "NM: error: Got invalid dimensions for input: causal_mask for the following indices\n index: 2 Got: 128 Expected: 1\n Please fix either the inputs or the model.", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[206], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mprefill\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtokens\u001b[49m\u001b[43m)\u001b[49m\n", + "Cell \u001b[0;32mIn[204], line 187\u001b[0m, in \u001b[0;36mModel.prefill\u001b[0;34m(self, tokens)\u001b[0m\n\u001b[1;32m 185\u001b[0m \u001b[38;5;66;03m# loop of singletoken engine for anything left over\u001b[39;00m\n\u001b[1;32m 186\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m tokens_processed \u001b[38;5;241m<\u001b[39m \u001b[38;5;28mlen\u001b[39m(tokens):\n\u001b[0;32m--> 187\u001b[0m logits, past_key_values \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdecode\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 188\u001b[0m \u001b[43m \u001b[49m\u001b[43mtokens\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtokens\u001b[49m\u001b[43m[\u001b[49m\u001b[43m:\u001b[49m\u001b[43mtokens_processed\u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 189\u001b[0m \u001b[43m \u001b[49m\u001b[43mpast_key_values\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpast_key_values\u001b[49m\n\u001b[1;32m 190\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 191\u001b[0m tokens_processed \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m\n\u001b[1;32m 193\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m logits\u001b[38;5;241m.\u001b[39mshape[\u001b[38;5;241m0\u001b[39m] \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m1\u001b[39m \u001b[38;5;66;03m# assert batch 1 right now\u001b[39;00m\n", + "Cell \u001b[0;32mIn[204], line 200\u001b[0m, in \u001b[0;36mModel.decode\u001b[0;34m(self, tokens, past_key_values)\u001b[0m\n\u001b[1;32m 197\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdecode\u001b[39m(\u001b[38;5;28mself\u001b[39m, tokens, past_key_values): \n\u001b[1;32m 198\u001b[0m engine_inputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mengine_inputs_for_decode(tokens)\n\u001b[0;32m--> 200\u001b[0m logits, past_key_values \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msingletoken_engine\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 201\u001b[0m \u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mengine_inputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 202\u001b[0m \u001b[43m \u001b[49m\u001b[43mpast_key_values\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpast_key_values\u001b[49m\n\u001b[1;32m 203\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 205\u001b[0m \u001b[38;5;66;03m# cleanup state (this is BAD - calls np.ascontiguous)\u001b[39;00m\n\u001b[1;32m 206\u001b[0m past_key_values \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mslice_past_key_values(past_key_values, \u001b[38;5;241m1\u001b[39m)\n", + "Cell \u001b[0;32mIn[204], line 54\u001b[0m, in \u001b[0;36mDecoderEngine.__call__\u001b[0;34m(self, inputs, past_key_values, val_inp)\u001b[0m\n\u001b[1;32m 50\u001b[0m inp \u001b[38;5;241m=\u001b[39m [past_key_values[name] \u001b[38;5;28;01mif\u001b[39;00m name\u001b[38;5;241m.\u001b[39mstartswith(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpast_key_values\u001b[39m\u001b[38;5;124m\"\u001b[39m) \n\u001b[1;32m 51\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m inputs[name] \u001b[38;5;28;01mfor\u001b[39;00m name \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mengine\u001b[38;5;241m.\u001b[39minput_names]\n\u001b[1;32m 53\u001b[0m \u001b[38;5;66;03m# run inference\u001b[39;00m\n\u001b[0;32m---> 54\u001b[0m logits, \u001b[38;5;241m*\u001b[39mkvs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mengine\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun\u001b[49m\u001b[43m(\u001b[49m\u001b[43minp\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[1;32m 55\u001b[0m past_key_values \u001b[38;5;241m=\u001b[39m {name: arr \u001b[38;5;28;01mfor\u001b[39;00m name, arr \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpast_onnx_inputs, kvs)}\n\u001b[1;32m 57\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m logits, past_key_values\n", + "File \u001b[0;32m~/.conda/envs/dscb/lib/python3.9/site-packages/deepsparse/engine.py:527\u001b[0m, in \u001b[0;36mEngine.run\u001b[0;34m(self, inp, val_inp)\u001b[0m\n\u001b[1;32m 524\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m val_inp:\n\u001b[1;32m 525\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_validate_inputs(inp)\n\u001b[0;32m--> 527\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_eng_net\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute_list_out\u001b[49m\u001b[43m(\u001b[49m\u001b[43minp\u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[0;31mRuntimeError\u001b[0m: NM: error: Got invalid dimensions for input: causal_mask for the following indices\n index: 2 Got: 128 Expected: 1\n Please fix either the inputs or the model." + ] + } + ], + "source": [ + "model.prefill(tokens)" + ] + }, + { + "cell_type": "code", + "execution_count": 168, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'input_ids': array([[[48658, 262, 1708, 2163, 329, 14492, 257, 12900, 261,\n", + " 44456, 8379, 25, 220, 628, 12900, 7, 77, 2599]]]), 'attention_mask': array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]]), 'causal_mask': array([[[[0, 0, 0, ..., 0, 0, 0],\n", + " [0, 0, 0, ..., 0, 0, 0],\n", + " [0, 0, 0, ..., 0, 0, 0],\n", + " ...,\n", + " [0, 0, 0, ..., 1, 0, 0],\n", + " [0, 0, 0, ..., 1, 1, 0],\n", + " [0, 0, 0, ..., 1, 1, 1]]]]), 'positions': array([[0]])}\n" + ] + }, + { + "ename": "RuntimeError", + "evalue": "NM: error: Invalid rank for input: input_ids Got: 3 Expected: 2 Please fix either the inputs or the model.", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[168], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mprefill\u001b[49m\u001b[43m(\u001b[49m\u001b[43minput_tokens\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43minput_ids\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n", + "Cell \u001b[0;32mIn[166], line 187\u001b[0m, in \u001b[0;36mModel.prefill\u001b[0;34m(self, tokens)\u001b[0m\n\u001b[1;32m 185\u001b[0m \u001b[38;5;66;03m# loop of singletoken engine for anything left over\u001b[39;00m\n\u001b[1;32m 186\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m tokens_processed \u001b[38;5;241m<\u001b[39m \u001b[38;5;28mlen\u001b[39m(tokens):\n\u001b[0;32m--> 187\u001b[0m logits, past_key_values \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdecode\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 188\u001b[0m \u001b[43m \u001b[49m\u001b[43mtokens\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtokens\u001b[49m\u001b[43m[\u001b[49m\u001b[43m:\u001b[49m\u001b[43mtokens_processed\u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 189\u001b[0m \u001b[43m \u001b[49m\u001b[43mpast_key_values\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpast_key_values\u001b[49m\n\u001b[1;32m 190\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 191\u001b[0m tokens_processed \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m\n\u001b[1;32m 193\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m logits\u001b[38;5;241m.\u001b[39mshape[\u001b[38;5;241m0\u001b[39m] \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m1\u001b[39m \u001b[38;5;66;03m# assert batch 1 right now\u001b[39;00m\n", + "Cell \u001b[0;32mIn[166], line 201\u001b[0m, in \u001b[0;36mModel.decode\u001b[0;34m(self, tokens, past_key_values)\u001b[0m\n\u001b[1;32m 198\u001b[0m engine_inputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mengine_inputs_for_decode(tokens)\n\u001b[1;32m 200\u001b[0m \u001b[38;5;28mprint\u001b[39m(engine_inputs)\n\u001b[0;32m--> 201\u001b[0m logits, past_key_values \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msingletoken_engine\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 202\u001b[0m \u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mengine_inputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 203\u001b[0m \u001b[43m \u001b[49m\u001b[43mpast_key_values\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpast_key_values\u001b[49m\n\u001b[1;32m 204\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 206\u001b[0m \u001b[38;5;66;03m# cleanup state (this is BAD - calls np.ascontiguous)\u001b[39;00m\n\u001b[1;32m 207\u001b[0m past_key_values \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mslice_past_key_values(past_key_values, \u001b[38;5;241m1\u001b[39m)\n", + "Cell \u001b[0;32mIn[146], line 54\u001b[0m, in \u001b[0;36mDecoderEngine.__call__\u001b[0;34m(self, inputs, past_key_values, val_inp)\u001b[0m\n\u001b[1;32m 50\u001b[0m inp \u001b[38;5;241m=\u001b[39m [past_key_values[name] \u001b[38;5;28;01mif\u001b[39;00m name\u001b[38;5;241m.\u001b[39mstartswith(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpast_key_values\u001b[39m\u001b[38;5;124m\"\u001b[39m) \n\u001b[1;32m 51\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m inputs[name] \u001b[38;5;28;01mfor\u001b[39;00m name \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mengine\u001b[38;5;241m.\u001b[39minput_names]\n\u001b[1;32m 53\u001b[0m \u001b[38;5;66;03m# run inference\u001b[39;00m\n\u001b[0;32m---> 54\u001b[0m logits, \u001b[38;5;241m*\u001b[39mkvs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mengine\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun\u001b[49m\u001b[43m(\u001b[49m\u001b[43minp\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[1;32m 55\u001b[0m past_key_values \u001b[38;5;241m=\u001b[39m {name: arr \u001b[38;5;28;01mfor\u001b[39;00m name, arr \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpast_onnx_names, kvs)}\n\u001b[1;32m 57\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m logits, past_key_values\n", + "File \u001b[0;32m~/.conda/envs/dscb/lib/python3.9/site-packages/deepsparse/engine.py:527\u001b[0m, in \u001b[0;36mEngine.run\u001b[0;34m(self, inp, val_inp)\u001b[0m\n\u001b[1;32m 524\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m val_inp:\n\u001b[1;32m 525\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_validate_inputs(inp)\n\u001b[0;32m--> 527\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_eng_net\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute_list_out\u001b[49m\u001b[43m(\u001b[49m\u001b[43minp\u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[0;31mRuntimeError\u001b[0m: NM: error: Invalid rank for input: input_ids Got: 3 Expected: 2 Please fix either the inputs or the model." + ] + } + ], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.17" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +}