mirror of
https://github.com/huggingface/text-generation-inference.git
synced 2025-09-10 20:04:52 +00:00
intermediate results from server
This commit is contained in:
rsnm2
parent
b8f7a0c690
commit
e7f3eac8d5
170
server-dev.ipynb
170
server-dev.ipynb
@ -21,10 +21,39 @@
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
"cell_type": "code",
|
"cell_type": "code",
|
||||||
"execution_count": 38,
|
"execution_count": 12,
|
||||||
|
"id": "f23bc085-94db-44b6-af42-fc8a05f2cf6a",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"ename": "SyntaxError",
|
||||||
|
"evalue": "invalid syntax (260114089.py, line 2)",
|
||||||
|
"output_type": "error",
|
||||||
|
"traceback": [
|
||||||
|
"\u001b[0;36m Cell \u001b[0;32mIn[12], line 2\u001b[0;36m\u001b[0m\n\u001b[0;31m b = (a = 5) < 5\u001b[0m\n\u001b[0m ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"a = None\n",
|
||||||
|
"b = (a = 5) < 5\n",
|
||||||
|
"print(b)\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 2,
|
||||||
"id": "631e94eb-cca0-438e-8936-6e8a87166d63",
|
"id": "631e94eb-cca0-438e-8936-6e8a87166d63",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stderr",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"2023-08-22 14:26:39 deepsparse.transformers WARNING The neuralmagic fork of transformers may not be installed. It can be installed via `pip install nm_transformers`\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
"source": [
|
"source": [
|
||||||
"from server.deepsparse.deepsparse_causal_lm import DeepSparseCausalLMBatch, DeepSparseCausalLM\n",
|
"from server.deepsparse.deepsparse_causal_lm import DeepSparseCausalLMBatch, DeepSparseCausalLM\n",
|
||||||
"from server.deepsparse.deepsparse_service import DeepSparseService\n",
|
"from server.deepsparse.deepsparse_service import DeepSparseService\n",
|
||||||
@ -44,7 +73,7 @@
|
|||||||
"output_type": "stream",
|
"output_type": "stream",
|
||||||
"text": [
|
"text": [
|
||||||
"Using pad_token, but it is not set yet.\n",
|
"Using pad_token, but it is not set yet.\n",
|
||||||
"2023-08-22 03:09:19 deepsparse.transformers.utils.helpers INFO Overwriting in-place the input shapes of the transformer model at /home/robertgshaw/.cache/sparsezoo/neuralmagic/codegen_mono-350m-bigpython_bigquery_thepile-base/model.onnx/model.onnx\n",
|
"2023-08-22 14:26:56 deepsparse.transformers.utils.helpers INFO Overwriting in-place the input shapes of the transformer model at /home/robertgshaw/.cache/sparsezoo/neuralmagic/codegen_mono-350m-bigpython_bigquery_thepile-base/model.onnx/model.onnx\n",
|
||||||
"DeepSparse, Copyright 2021-present / Neuralmagic, Inc. version: 1.6.0.20230815 COMMUNITY | (134dba40) (release) (optimized) (system=avx2, binary=avx2)\n"
|
"DeepSparse, Copyright 2021-present / Neuralmagic, Inc. version: 1.6.0.20230815 COMMUNITY | (134dba40) (release) (optimized) (system=avx2, binary=avx2)\n"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
@ -67,7 +96,7 @@
|
|||||||
"name": "stderr",
|
"name": "stderr",
|
||||||
"output_type": "stream",
|
"output_type": "stream",
|
||||||
"text": [
|
"text": [
|
||||||
"2023-08-22 03:09:45 deepsparse.transformers.utils.helpers INFO Overwriting in-place the input shapes of the transformer model at /home/robertgshaw/.cache/sparsezoo/neuralmagic/codegen_mono-350m-bigpython_bigquery_thepile-base/model.onnx/model.onnx\n"
|
"2023-08-22 14:27:21 deepsparse.transformers.utils.helpers INFO Overwriting in-place the input shapes of the transformer model at /home/robertgshaw/.cache/sparsezoo/neuralmagic/codegen_mono-350m-bigpython_bigquery_thepile-base/model.onnx/model.onnx\n"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@ -100,7 +129,7 @@
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
"cell_type": "code",
|
"cell_type": "code",
|
||||||
"execution_count": 31,
|
"execution_count": 4,
|
||||||
"id": "85ce9aab-1a56-4b6f-a82b-4e91d52290b7",
|
"id": "85ce9aab-1a56-4b6f-a82b-4e91d52290b7",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
@ -136,10 +165,25 @@
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
"cell_type": "code",
|
"cell_type": "code",
|
||||||
"execution_count": 63,
|
"execution_count": 5,
|
||||||
"id": "d2441753-fe2a-45c0-ad80-135b6207947d",
|
"id": "d2441753-fe2a-45c0-ad80-135b6207947d",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [
|
||||||
|
{
|
||||||
|
"ename": "NameError",
|
||||||
|
"evalue": "name 'Batch' is not defined",
|
||||||
|
"output_type": "error",
|
||||||
|
"traceback": [
|
||||||
|
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
||||||
|
"\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",
|
||||||
|
"Cell \u001b[0;32mIn[5], line 4\u001b[0m\n\u001b[1;32m 1\u001b[0m service\u001b[38;5;241m.\u001b[39mClearCache()\n\u001b[1;32m 3\u001b[0m \u001b[38;5;66;03m# prefill queue\u001b[39;00m\n\u001b[0;32m----> 4\u001b[0m prefill_queue \u001b[38;5;241m=\u001b[39m \u001b[43mPrefillQueue\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprompts\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 6\u001b[0m \u001b[38;5;66;03m# cached batches\u001b[39;00m\n\u001b[1;32m 7\u001b[0m cached_batches \u001b[38;5;241m=\u001b[39m []\n",
|
||||||
|
"Cell \u001b[0;32mIn[4], line 17\u001b[0m, in \u001b[0;36mPrefillQueue.__init__\u001b[0;34m(self, prompts)\u001b[0m\n\u001b[1;32m 16\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, prompts):\n\u001b[0;32m---> 17\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mqueue \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m 18\u001b[0m idx: PrefillRequest(batch\u001b[38;5;241m=\u001b[39mmake_batch(\u001b[38;5;28mid\u001b[39m\u001b[38;5;241m=\u001b[39midx, prompt\u001b[38;5;241m=\u001b[39mprompt))\n\u001b[1;32m 19\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m idx, prompt \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28menumerate\u001b[39m(prompts)\n\u001b[1;32m 20\u001b[0m }\n",
|
||||||
|
"Cell \u001b[0;32mIn[4], line 18\u001b[0m, in \u001b[0;36m<dictcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 16\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, prompts):\n\u001b[1;32m 17\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mqueue \u001b[38;5;241m=\u001b[39m {\n\u001b[0;32m---> 18\u001b[0m idx: PrefillRequest(batch\u001b[38;5;241m=\u001b[39m\u001b[43mmake_batch\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mid\u001b[39;49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43midx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mprompt\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mprompt\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[1;32m 19\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m idx, prompt \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28menumerate\u001b[39m(prompts)\n\u001b[1;32m 20\u001b[0m }\n",
|
||||||
|
"Cell \u001b[0;32mIn[4], line 10\u001b[0m, in \u001b[0;36mmake_batch\u001b[0;34m(id, prompt)\u001b[0m\n\u001b[1;32m 9\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mmake_batch\u001b[39m(\u001b[38;5;28mid\u001b[39m, prompt):\n\u001b[0;32m---> 10\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mBatch\u001b[49m(\n\u001b[1;32m 11\u001b[0m \u001b[38;5;28mid\u001b[39m\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mid\u001b[39m,\n\u001b[1;32m 12\u001b[0m requests\u001b[38;5;241m=\u001b[39m[Request(\u001b[38;5;28mid\u001b[39m\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mid\u001b[39m, prompt\u001b[38;5;241m=\u001b[39mprompt)]\n\u001b[1;32m 13\u001b[0m )\n",
|
||||||
|
"\u001b[0;31mNameError\u001b[0m: name 'Batch' is not defined"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
"source": [
|
"source": [
|
||||||
"service.ClearCache()\n",
|
"service.ClearCache()\n",
|
||||||
"\n",
|
"\n",
|
||||||
@ -213,118 +257,10 @@
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
"cell_type": "code",
|
"cell_type": "code",
|
||||||
"execution_count": 64,
|
"execution_count": null,
|
||||||
"id": "dd6bcc43-63ef-4f92-a960-74e33b86dc97",
|
"id": "dd6bcc43-63ef-4f92-a960-74e33b86dc97",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [
|
"outputs": [],
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"Request 0 is done!\n",
|
|
||||||
"Request 1 is done!\n",
|
|
||||||
"Request 3 is done!\n",
|
|
||||||
"Request 2 is done!\n",
|
|
||||||
"All Requests Done!\n",
|
|
||||||
"\n",
|
|
||||||
"\n",
|
|
||||||
"INDEX = 0:\n",
|
|
||||||
"Finish the following function for computing a fibonacci sequence: \n",
|
|
||||||
"\n",
|
|
||||||
" fib(n):\n",
|
|
||||||
"\n",
|
|
||||||
" if n == 0:\n",
|
|
||||||
" return 0\n",
|
|
||||||
" elif n == 1:\n",
|
|
||||||
" return 1\n",
|
|
||||||
" else:\n",
|
|
||||||
" return fib(n-1) + fib(n-2)\n",
|
|
||||||
"\n",
|
|
||||||
"# Call the function.\n",
|
|
||||||
"print(fib(5))\n",
|
|
||||||
"\n",
|
|
||||||
"# This code is contributed by Nikhil Kumar Singh(nickzuck_007)\n",
|
|
||||||
"\n",
|
|
||||||
"\n",
|
|
||||||
"\n",
|
|
||||||
"INDEX = 1:\n",
|
|
||||||
"Write a function for filtering a list of integers to include only positive numbers:\n",
|
|
||||||
"\n",
|
|
||||||
"filter(lst):\n",
|
|
||||||
"\n",
|
|
||||||
"lst = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]\n",
|
|
||||||
"\n",
|
|
||||||
"def filter_positive(lst):\n",
|
|
||||||
" return [num for num in lst if num > 0]\n",
|
|
||||||
"\n",
|
|
||||||
"print(filter_positive(lst))\n",
|
|
||||||
"\n",
|
|
||||||
"# filter_positive([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])\n",
|
|
||||||
"\n",
|
|
||||||
"# filter_positive([1, 2, 3, 4, 5\n",
|
|
||||||
"\n",
|
|
||||||
"\n",
|
|
||||||
"INDEX = 2:\n",
|
|
||||||
"Write a function for reversing a string:\n",
|
|
||||||
"\n",
|
|
||||||
"def reverse_string(s):\n",
|
|
||||||
" return s[::-1]\n",
|
|
||||||
"\n",
|
|
||||||
"# Test\n",
|
|
||||||
"print(reverse_string(\"hello\"))\n",
|
|
||||||
"print(reverse_string(\"\"))\n",
|
|
||||||
"print(reverse_string(\"a\"))\n",
|
|
||||||
"print(reverse_string(\"\"))\n",
|
|
||||||
"print(reverse_string(\"\"))\n",
|
|
||||||
"print(reverse_string(\"\"))\n",
|
|
||||||
"print(reverse_string(\"\"))\n",
|
|
||||||
"print(reverse_string(\"\"))\n",
|
|
||||||
"print(reverse_string(\"\"))\n",
|
|
||||||
"print(reverse_string(\"\"))\n",
|
|
||||||
"print(reverse_string(\"\"))\n",
|
|
||||||
"print(reverse_string(\"\n",
|
|
||||||
"\n",
|
|
||||||
"\n",
|
|
||||||
"INDEX = 3:\n",
|
|
||||||
"Write a function for checking if a word if a palindrome:\n",
|
|
||||||
"\n",
|
|
||||||
"def is_palindrome(word):\n",
|
|
||||||
" return word == word[::-1]\n",
|
|
||||||
"\n",
|
|
||||||
"# Test\n",
|
|
||||||
"print(is_palindrome(\"racecar\"))\n",
|
|
||||||
"print(is_palindrome(\"racecar\"))\n",
|
|
||||||
"print(is_palindrome(\"racecar\"))\n",
|
|
||||||
"print(is_palindrome(\"racecar\"))\n",
|
|
||||||
"print(is_palindrome(\"racecar\"))\n",
|
|
||||||
"print(is_palindrome(\"racecar\"))\n",
|
|
||||||
"print(is_palindrome(\"racecar\"))\n",
|
|
||||||
"print(\n",
|
|
||||||
"\n",
|
|
||||||
"\n",
|
|
||||||
"INDEX = 4:\n",
|
|
||||||
"Write a function for sorting an array of integers:\n",
|
|
||||||
"\n",
|
|
||||||
"def merge_sort(arr):\n",
|
|
||||||
" if len(arr) <= 1:\n",
|
|
||||||
" return arr\n",
|
|
||||||
" mid = len(arr) // 2\n",
|
|
||||||
" left = arr[:mid]\n",
|
|
||||||
" right = arr[mid:]\n",
|
|
||||||
" left = merge_sort(left)\n",
|
|
||||||
" right = merge_sort(right)\n",
|
|
||||||
" return merge(left, right)\n",
|
|
||||||
"\n",
|
|
||||||
"def merge(left, right):\n",
|
|
||||||
" result = []\n",
|
|
||||||
" while len(left) > 0 and len(right) > 0:\n",
|
|
||||||
" if left[0]\n",
|
|
||||||
"\n",
|
|
||||||
"\n",
|
|
||||||
"[CachedBatch(batch_id=0, request_ids=[4])]\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
"source": [
|
||||||
"# run a few decodes\n",
|
"# run a few decodes\n",
|
||||||
"for _ in range(100):\n",
|
"for _ in range(100):\n",
|
||||||
|
|||||||
@ -5,6 +5,7 @@ from typing import List, Optional
|
|||||||
class Request:
|
class Request:
|
||||||
id: int
|
id: int
|
||||||
prompt: str
|
prompt: str
|
||||||
|
max_generated_tokens: int
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class Batch:
|
class Batch:
|
||||||
@ -16,6 +17,9 @@ class CachedBatch:
|
|||||||
batch_id: int
|
batch_id: int
|
||||||
request_ids: List[int]
|
request_ids: List[int]
|
||||||
|
|
||||||
|
def __len__(self):
|
||||||
|
return len(self.request_ids)
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class Generation:
|
class Generation:
|
||||||
request_id: int
|
request_id: int
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user