diff --git a/server-dev.ipynb b/server-dev.ipynb index 782920c8..b8f684a8 100644 --- a/server-dev.ipynb +++ b/server-dev.ipynb @@ -21,10 +21,39 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 12, + "id": "f23bc085-94db-44b6-af42-fc8a05f2cf6a", + "metadata": {}, + "outputs": [ + { + "ename": "SyntaxError", + "evalue": "invalid syntax (260114089.py, line 2)", + "output_type": "error", + "traceback": [ + "\u001b[0;36m Cell \u001b[0;32mIn[12], line 2\u001b[0;36m\u001b[0m\n\u001b[0;31m b = (a = 5) < 5\u001b[0m\n\u001b[0m ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n" + ] + } + ], + "source": [ + "a = None\n", + "b = (a = 5) < 5\n", + "print(b)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, "id": "631e94eb-cca0-438e-8936-6e8a87166d63", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-22 14:26:39 deepsparse.transformers WARNING The neuralmagic fork of transformers may not be installed. It can be installed via `pip install nm_transformers`\n" + ] + } + ], "source": [ "from server.deepsparse.deepsparse_causal_lm import DeepSparseCausalLMBatch, DeepSparseCausalLM\n", "from server.deepsparse.deepsparse_service import DeepSparseService\n", @@ -44,7 +73,7 @@ "output_type": "stream", "text": [ "Using pad_token, but it is not set yet.\n", - "2023-08-22 03:09:19 deepsparse.transformers.utils.helpers INFO Overwriting in-place the input shapes of the transformer model at /home/robertgshaw/.cache/sparsezoo/neuralmagic/codegen_mono-350m-bigpython_bigquery_thepile-base/model.onnx/model.onnx\n", + "2023-08-22 14:26:56 deepsparse.transformers.utils.helpers INFO Overwriting in-place the input shapes of the transformer model at /home/robertgshaw/.cache/sparsezoo/neuralmagic/codegen_mono-350m-bigpython_bigquery_thepile-base/model.onnx/model.onnx\n", "DeepSparse, Copyright 2021-present / Neuralmagic, Inc. version: 1.6.0.20230815 COMMUNITY | (134dba40) (release) (optimized) (system=avx2, binary=avx2)\n" ] }, @@ -67,7 +96,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "2023-08-22 03:09:45 deepsparse.transformers.utils.helpers INFO Overwriting in-place the input shapes of the transformer model at /home/robertgshaw/.cache/sparsezoo/neuralmagic/codegen_mono-350m-bigpython_bigquery_thepile-base/model.onnx/model.onnx\n" + "2023-08-22 14:27:21 deepsparse.transformers.utils.helpers INFO Overwriting in-place the input shapes of the transformer model at /home/robertgshaw/.cache/sparsezoo/neuralmagic/codegen_mono-350m-bigpython_bigquery_thepile-base/model.onnx/model.onnx\n" ] }, { @@ -100,7 +129,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 4, "id": "85ce9aab-1a56-4b6f-a82b-4e91d52290b7", "metadata": {}, "outputs": [], @@ -136,10 +165,25 @@ }, { "cell_type": "code", - "execution_count": 63, + "execution_count": 5, "id": "d2441753-fe2a-45c0-ad80-135b6207947d", "metadata": {}, - "outputs": [], + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'Batch' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[5], line 4\u001b[0m\n\u001b[1;32m 1\u001b[0m service\u001b[38;5;241m.\u001b[39mClearCache()\n\u001b[1;32m 3\u001b[0m \u001b[38;5;66;03m# prefill queue\u001b[39;00m\n\u001b[0;32m----> 4\u001b[0m prefill_queue \u001b[38;5;241m=\u001b[39m \u001b[43mPrefillQueue\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprompts\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 6\u001b[0m \u001b[38;5;66;03m# cached batches\u001b[39;00m\n\u001b[1;32m 7\u001b[0m cached_batches \u001b[38;5;241m=\u001b[39m []\n", + "Cell \u001b[0;32mIn[4], line 17\u001b[0m, in \u001b[0;36mPrefillQueue.__init__\u001b[0;34m(self, prompts)\u001b[0m\n\u001b[1;32m 16\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, prompts):\n\u001b[0;32m---> 17\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mqueue \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m 18\u001b[0m idx: PrefillRequest(batch\u001b[38;5;241m=\u001b[39mmake_batch(\u001b[38;5;28mid\u001b[39m\u001b[38;5;241m=\u001b[39midx, prompt\u001b[38;5;241m=\u001b[39mprompt))\n\u001b[1;32m 19\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m idx, prompt \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28menumerate\u001b[39m(prompts)\n\u001b[1;32m 20\u001b[0m }\n", + "Cell \u001b[0;32mIn[4], line 18\u001b[0m, in \u001b[0;36m\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 16\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, prompts):\n\u001b[1;32m 17\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mqueue \u001b[38;5;241m=\u001b[39m {\n\u001b[0;32m---> 18\u001b[0m idx: PrefillRequest(batch\u001b[38;5;241m=\u001b[39m\u001b[43mmake_batch\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mid\u001b[39;49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43midx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mprompt\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mprompt\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[1;32m 19\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m idx, prompt \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28menumerate\u001b[39m(prompts)\n\u001b[1;32m 20\u001b[0m }\n", + "Cell \u001b[0;32mIn[4], line 10\u001b[0m, in \u001b[0;36mmake_batch\u001b[0;34m(id, prompt)\u001b[0m\n\u001b[1;32m 9\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mmake_batch\u001b[39m(\u001b[38;5;28mid\u001b[39m, prompt):\n\u001b[0;32m---> 10\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mBatch\u001b[49m(\n\u001b[1;32m 11\u001b[0m \u001b[38;5;28mid\u001b[39m\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mid\u001b[39m,\n\u001b[1;32m 12\u001b[0m requests\u001b[38;5;241m=\u001b[39m[Request(\u001b[38;5;28mid\u001b[39m\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mid\u001b[39m, prompt\u001b[38;5;241m=\u001b[39mprompt)]\n\u001b[1;32m 13\u001b[0m )\n", + "\u001b[0;31mNameError\u001b[0m: name 'Batch' is not defined" + ] + } + ], "source": [ "service.ClearCache()\n", "\n", @@ -213,118 +257,10 @@ }, { "cell_type": "code", - "execution_count": 64, + "execution_count": null, "id": "dd6bcc43-63ef-4f92-a960-74e33b86dc97", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Request 0 is done!\n", - "Request 1 is done!\n", - "Request 3 is done!\n", - "Request 2 is done!\n", - "All Requests Done!\n", - "\n", - "\n", - "INDEX = 0:\n", - "Finish the following function for computing a fibonacci sequence: \n", - "\n", - " fib(n):\n", - "\n", - " if n == 0:\n", - " return 0\n", - " elif n == 1:\n", - " return 1\n", - " else:\n", - " return fib(n-1) + fib(n-2)\n", - "\n", - "# Call the function.\n", - "print(fib(5))\n", - "\n", - "# This code is contributed by Nikhil Kumar Singh(nickzuck_007)\n", - "\n", - "\n", - "\n", - "INDEX = 1:\n", - "Write a function for filtering a list of integers to include only positive numbers:\n", - "\n", - "filter(lst):\n", - "\n", - "lst = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]\n", - "\n", - "def filter_positive(lst):\n", - " return [num for num in lst if num > 0]\n", - "\n", - "print(filter_positive(lst))\n", - "\n", - "# filter_positive([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])\n", - "\n", - "# filter_positive([1, 2, 3, 4, 5\n", - "\n", - "\n", - "INDEX = 2:\n", - "Write a function for reversing a string:\n", - "\n", - "def reverse_string(s):\n", - " return s[::-1]\n", - "\n", - "# Test\n", - "print(reverse_string(\"hello\"))\n", - "print(reverse_string(\"\"))\n", - "print(reverse_string(\"a\"))\n", - "print(reverse_string(\"\"))\n", - "print(reverse_string(\"\"))\n", - "print(reverse_string(\"\"))\n", - "print(reverse_string(\"\"))\n", - "print(reverse_string(\"\"))\n", - "print(reverse_string(\"\"))\n", - "print(reverse_string(\"\"))\n", - "print(reverse_string(\"\"))\n", - "print(reverse_string(\"\n", - "\n", - "\n", - "INDEX = 3:\n", - "Write a function for checking if a word if a palindrome:\n", - "\n", - "def is_palindrome(word):\n", - " return word == word[::-1]\n", - "\n", - "# Test\n", - "print(is_palindrome(\"racecar\"))\n", - "print(is_palindrome(\"racecar\"))\n", - "print(is_palindrome(\"racecar\"))\n", - "print(is_palindrome(\"racecar\"))\n", - "print(is_palindrome(\"racecar\"))\n", - "print(is_palindrome(\"racecar\"))\n", - "print(is_palindrome(\"racecar\"))\n", - "print(\n", - "\n", - "\n", - "INDEX = 4:\n", - "Write a function for sorting an array of integers:\n", - "\n", - "def merge_sort(arr):\n", - " if len(arr) <= 1:\n", - " return arr\n", - " mid = len(arr) // 2\n", - " left = arr[:mid]\n", - " right = arr[mid:]\n", - " left = merge_sort(left)\n", - " right = merge_sort(right)\n", - " return merge(left, right)\n", - "\n", - "def merge(left, right):\n", - " result = []\n", - " while len(left) > 0 and len(right) > 0:\n", - " if left[0]\n", - "\n", - "\n", - "[CachedBatch(batch_id=0, request_ids=[4])]\n" - ] - } - ], + "outputs": [], "source": [ "# run a few decodes\n", "for _ in range(100):\n", diff --git a/server/deepsparse/deepsparse_requests.py b/server/deepsparse/deepsparse_requests.py index 524793b0..430f3473 100644 --- a/server/deepsparse/deepsparse_requests.py +++ b/server/deepsparse/deepsparse_requests.py @@ -5,6 +5,7 @@ from typing import List, Optional class Request: id: int prompt: str + max_generated_tokens: int @dataclass class Batch: @@ -16,6 +17,9 @@ class CachedBatch: batch_id: int request_ids: List[int] + def __len__(self): + return len(self.request_ids) + @dataclass class Generation: request_id: int