mirror of
https://github.com/huggingface/text-generation-inference.git
synced 2025-09-11 04:14:52 +00:00
updated
This commit is contained in:
rsnm2
parent
bd5fcf6f13
commit
21f037939b
68
README.md
68
README.md
@ -1,71 +1,11 @@
|
|||||||
### Local install
|
Install:
|
||||||
|
|
||||||
You can also opt to install `text-generation-inference` locally.
|
|
||||||
|
|
||||||
First [install Rust](https://rustup.rs/):
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
|
pip3 install deepsparse-nightly[transformer] fastapi uvicorn
|
||||||
```
|
```
|
||||||
|
|
||||||
Install conda:
|
Download model:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
curl https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc | gpg --dearmor > conda.gpg
|
sparsezoo.download zoo:nlg/text_generation/codegen_multi-350m/pytorch/huggingface/bigquery_thepile/base_quant-none --save-dir codegen-quant
|
||||||
sudo install -o root -g root -m 644 conda.gpg /usr/share/keyrings/conda-archive-keyring.gpg
|
|
||||||
gpg --keyring /usr/share/keyrings/conda-archive-keyring.gpg --no-default-keyring --fingerprint 34161F5BF5EB1D4BFBBB8F0A8AEB4F8B29D82806
|
|
||||||
echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" | sudo tee -a /etc/apt/sources.list.d/conda.list
|
|
||||||
sudo apt update && sudo apt install conda -y
|
|
||||||
source /opt/conda/etc/profile.d/conda.sh
|
|
||||||
conda -V
|
|
||||||
```
|
|
||||||
|
|
||||||
Create Env:
|
|
||||||
|
|
||||||
```shell
|
|
||||||
conda create -n dscb python=3.9
|
|
||||||
conda activate dscb
|
|
||||||
```
|
|
||||||
|
|
||||||
Install PROTOC
|
|
||||||
```shell
|
|
||||||
PROTOC_ZIP=protoc-21.12-linux-x86_64.zip
|
|
||||||
curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v21.12/$PROTOC_ZIP
|
|
||||||
sudo unzip -o $PROTOC_ZIP -d /usr/local bin/protoc
|
|
||||||
sudo unzip -o $PROTOC_ZIP -d /usr/local 'include/*'
|
|
||||||
rm -f $PROTOC_ZIP
|
|
||||||
```
|
|
||||||
|
|
||||||
You might need to install these:
|
|
||||||
```shell
|
|
||||||
sudo apt-get install libssl-dev gcc -y
|
|
||||||
sudo apt-get install pkg-config
|
|
||||||
```
|
|
||||||
|
|
||||||
Install DeepSparse:
|
|
||||||
```shell
|
|
||||||
pip install deepsparse-nightly[transformers]
|
|
||||||
```
|
|
||||||
|
|
||||||
Install Server
|
|
||||||
```shell
|
|
||||||
make install-server
|
|
||||||
```
|
|
||||||
|
|
||||||
Launch Server
|
|
||||||
```shell
|
|
||||||
python3 server/text_generation_server/cli.py download-weights bigscience/bloom-560m
|
|
||||||
python3 server/text_generation_server/cli.py serve bigscience/bloom-560m
|
|
||||||
```
|
|
||||||
|
|
||||||
Launch Router
|
|
||||||
```shell
|
|
||||||
make router-dev
|
|
||||||
```
|
|
||||||
|
|
||||||
Install FastAPI/Uvicorn
|
|
||||||
|
|
||||||
```shell
|
|
||||||
pip install fastapi
|
|
||||||
pip install "uvicorn[standard]"
|
|
||||||
```
|
```
|
||||||
@ -1,5 +1,4 @@
|
|||||||
import fastapi, uvicorn
|
import fastapi, uvicorn
|
||||||
from sse_starlette.sse import EventSourceResponse
|
|
||||||
from contextlib import asynccontextmanager
|
from contextlib import asynccontextmanager
|
||||||
|
|
||||||
from threading import Thread
|
from threading import Thread
|
||||||
@ -9,15 +8,20 @@ from typing import Optional
|
|||||||
from router import DeepSparseRouter, batching_task
|
from router import DeepSparseRouter, batching_task
|
||||||
from utils import GenerateRequestInputs, GenerateRequestOutputs, GenerateRequest
|
from utils import GenerateRequestInputs, GenerateRequestOutputs, GenerateRequest
|
||||||
|
|
||||||
TOKENIZER_PATH = "/home/robertgshaw/.cache/sparsezoo/neuralmagic/codegen_mono-350m-bigpython_bigquery_thepile-base/deployment"
|
import argparse
|
||||||
MODEL_PATH = "/home/robertgshaw/.cache/sparsezoo/neuralmagic/codegen_mono-350m-bigpython_bigquery_thepile-base/model.onnx/model.onnx"
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--deployment-dir", type=str)
|
||||||
|
|
||||||
|
args = parser.parse_args()
|
||||||
|
deployment_dir = args.deployment_dir
|
||||||
|
model_path = deployment_dir + "/model.onnx"
|
||||||
|
|
||||||
artifacts = {}
|
artifacts = {}
|
||||||
|
|
||||||
@asynccontextmanager
|
@asynccontextmanager
|
||||||
async def lifespan(app: fastapi.FastAPI):
|
async def lifespan(app: fastapi.FastAPI):
|
||||||
print("\n-------------------- Building Router --------------------\n")
|
print("\n-------------------- Building Router --------------------\n")
|
||||||
artifacts["router"] = DeepSparseRouter(model_path=MODEL_PATH, tokenizer_path=TOKENIZER_PATH)
|
artifacts["router"] = DeepSparseRouter(model_path=model_path, tokenizer_path=deployment_dir)
|
||||||
|
|
||||||
print("\n-------------------- Starting Batching Task --------------------\n")
|
print("\n-------------------- Starting Batching Task --------------------\n")
|
||||||
batching_thread = Thread(target=batching_task, args=[artifacts["router"]])
|
batching_thread = Thread(target=batching_task, args=[artifacts["router"]])
|
||||||
|
|||||||
@ -7,7 +7,7 @@ from typing import Optional, List, Dict
|
|||||||
from deepsparse import Context
|
from deepsparse import Context
|
||||||
from deepsparse.engine import LIB
|
from deepsparse.engine import LIB
|
||||||
from deepsparse.pipeline import DEEPSPARSE_ENGINE, create_engine
|
from deepsparse.pipeline import DEEPSPARSE_ENGINE, create_engine
|
||||||
from deepsparse.transformers.utils.helpers import overwrite_onnx_model_inputs, create_causal_mask
|
from deepsparse.transformers.utils.helpers import overwrite_onnx_model_inputs_for_kv_cache_models, create_causal_mask
|
||||||
|
|
||||||
PAST_KEY_VALUES_NAME = "past_key_values"
|
PAST_KEY_VALUES_NAME = "past_key_values"
|
||||||
|
|
||||||
@ -27,7 +27,7 @@ class DeepSparseDecoderEngine:
|
|||||||
):
|
):
|
||||||
|
|
||||||
# setup ONNX graph
|
# setup ONNX graph
|
||||||
onnx_file_path, cached_outputs, data_type = overwrite_onnx_model_inputs(
|
onnx_file_path, cached_outputs, data_type = overwrite_onnx_model_inputs_for_kv_cache_models(
|
||||||
onnx_file_path=onnx_file_path,
|
onnx_file_path=onnx_file_path,
|
||||||
batch_size=1,
|
batch_size=1,
|
||||||
sequence_length=sequence_length,
|
sequence_length=sequence_length,
|
||||||
|
|||||||
247
server-dev.ipynb
247
server-dev.ipynb
@ -2,198 +2,53 @@
|
|||||||
"cells": [
|
"cells": [
|
||||||
{
|
{
|
||||||
"cell_type": "code",
|
"cell_type": "code",
|
||||||
"execution_count": 1,
|
"execution_count": 7,
|
||||||
"id": "2ab30adb-ca8a-4ca3-9cbc-dcae6e244754",
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"%reload_ext autoreload\n",
|
|
||||||
"%autoreload 2"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 108,
|
|
||||||
"id": "b5fd745b-6681-4e3d-81f2-2c721fde9318",
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"data": {
|
|
||||||
"text/plain": [
|
|
||||||
"'http://127.0.0.1:5543/generate_stream'"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"execution_count": 108,
|
|
||||||
"metadata": {},
|
|
||||||
"output_type": "execute_result"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"url"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 110,
|
|
||||||
"id": "4a6b2bf7-aa4e-45a3-b6c3-5f0e20b6a0e0",
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"ename": "KeyboardInterrupt",
|
|
||||||
"evalue": "",
|
|
||||||
"output_type": "error",
|
|
||||||
"traceback": [
|
|
||||||
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
|
||||||
"\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)",
|
|
||||||
"Cell \u001b[0;32mIn[110], line 16\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01msseclient\u001b[39;00m\n\u001b[1;32m 3\u001b[0m obj \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m 4\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124minputs\u001b[39m\u001b[38;5;124m\"\u001b[39m: sequence,\n\u001b[1;32m 5\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mgeneration_parameters\u001b[39m\u001b[38;5;124m\"\u001b[39m: {\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 13\u001b[0m }\n\u001b[1;32m 14\u001b[0m }\n\u001b[0;32m---> 16\u001b[0m stream_response \u001b[38;5;241m=\u001b[39m \u001b[43mrequests\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpost\u001b[49m\u001b[43m(\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mjson\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mobj\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstream\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[1;32m 18\u001b[0m client \u001b[38;5;241m=\u001b[39m sseclient\u001b[38;5;241m.\u001b[39mSSEClient(stream_response)\n\u001b[1;32m 20\u001b[0m \u001b[38;5;66;03m# Loop forever (while connection \"open\")\u001b[39;00m\n",
|
|
||||||
"File \u001b[0;32m~/.conda/envs/dscb/lib/python3.9/site-packages/requests/api.py:115\u001b[0m, in \u001b[0;36mpost\u001b[0;34m(url, data, json, **kwargs)\u001b[0m\n\u001b[1;32m 103\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mpost\u001b[39m(url, data\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, json\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 104\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124mr\u001b[39m\u001b[38;5;124;03m\"\"\"Sends a POST request.\u001b[39;00m\n\u001b[1;32m 105\u001b[0m \n\u001b[1;32m 106\u001b[0m \u001b[38;5;124;03m :param url: URL for the new :class:`Request` object.\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 112\u001b[0m \u001b[38;5;124;03m :rtype: requests.Response\u001b[39;00m\n\u001b[1;32m 113\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 115\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mpost\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mjson\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mjson\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
|
|
||||||
"File \u001b[0;32m~/.conda/envs/dscb/lib/python3.9/site-packages/requests/api.py:59\u001b[0m, in \u001b[0;36mrequest\u001b[0;34m(method, url, **kwargs)\u001b[0m\n\u001b[1;32m 55\u001b[0m \u001b[38;5;66;03m# By using the 'with' statement we are sure the session is closed, thus we\u001b[39;00m\n\u001b[1;32m 56\u001b[0m \u001b[38;5;66;03m# avoid leaving sockets open which can trigger a ResourceWarning in some\u001b[39;00m\n\u001b[1;32m 57\u001b[0m \u001b[38;5;66;03m# cases, and look like a memory leak in others.\u001b[39;00m\n\u001b[1;32m 58\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m sessions\u001b[38;5;241m.\u001b[39mSession() \u001b[38;5;28;01mas\u001b[39;00m session:\n\u001b[0;32m---> 59\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43msession\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmethod\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
|
|
||||||
"File \u001b[0;32m~/.conda/envs/dscb/lib/python3.9/site-packages/requests/sessions.py:589\u001b[0m, in \u001b[0;36mSession.request\u001b[0;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[0m\n\u001b[1;32m 584\u001b[0m send_kwargs \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m 585\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtimeout\u001b[39m\u001b[38;5;124m\"\u001b[39m: timeout,\n\u001b[1;32m 586\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mallow_redirects\u001b[39m\u001b[38;5;124m\"\u001b[39m: allow_redirects,\n\u001b[1;32m 587\u001b[0m }\n\u001b[1;32m 588\u001b[0m send_kwargs\u001b[38;5;241m.\u001b[39mupdate(settings)\n\u001b[0;32m--> 589\u001b[0m resp \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprep\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43msend_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 591\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m resp\n",
|
|
||||||
"File \u001b[0;32m~/.conda/envs/dscb/lib/python3.9/site-packages/requests/sessions.py:703\u001b[0m, in \u001b[0;36mSession.send\u001b[0;34m(self, request, **kwargs)\u001b[0m\n\u001b[1;32m 700\u001b[0m start \u001b[38;5;241m=\u001b[39m preferred_clock()\n\u001b[1;32m 702\u001b[0m \u001b[38;5;66;03m# Send the request\u001b[39;00m\n\u001b[0;32m--> 703\u001b[0m r \u001b[38;5;241m=\u001b[39m \u001b[43madapter\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 705\u001b[0m \u001b[38;5;66;03m# Total elapsed time of the request (approximately)\u001b[39;00m\n\u001b[1;32m 706\u001b[0m elapsed \u001b[38;5;241m=\u001b[39m preferred_clock() \u001b[38;5;241m-\u001b[39m start\n",
|
|
||||||
"File \u001b[0;32m~/.conda/envs/dscb/lib/python3.9/site-packages/requests/adapters.py:486\u001b[0m, in \u001b[0;36mHTTPAdapter.send\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m 483\u001b[0m timeout \u001b[38;5;241m=\u001b[39m TimeoutSauce(connect\u001b[38;5;241m=\u001b[39mtimeout, read\u001b[38;5;241m=\u001b[39mtimeout)\n\u001b[1;32m 485\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 486\u001b[0m resp \u001b[38;5;241m=\u001b[39m \u001b[43mconn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 487\u001b[0m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 488\u001b[0m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 489\u001b[0m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 490\u001b[0m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 491\u001b[0m \u001b[43m \u001b[49m\u001b[43mredirect\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 492\u001b[0m \u001b[43m \u001b[49m\u001b[43massert_same_host\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 493\u001b[0m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 494\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 495\u001b[0m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmax_retries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 496\u001b[0m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 497\u001b[0m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 498\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 500\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (ProtocolError, \u001b[38;5;167;01mOSError\u001b[39;00m) \u001b[38;5;28;01mas\u001b[39;00m err:\n\u001b[1;32m 501\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m(err, request\u001b[38;5;241m=\u001b[39mrequest)\n",
|
|
||||||
"File \u001b[0;32m~/.conda/envs/dscb/lib/python3.9/site-packages/urllib3/connectionpool.py:790\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[0m\n\u001b[1;32m 787\u001b[0m response_conn \u001b[38;5;241m=\u001b[39m conn \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m release_conn \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 789\u001b[0m \u001b[38;5;66;03m# Make the request on the HTTPConnection object\u001b[39;00m\n\u001b[0;32m--> 790\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_make_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 791\u001b[0m \u001b[43m \u001b[49m\u001b[43mconn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 792\u001b[0m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 793\u001b[0m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 794\u001b[0m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 795\u001b[0m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 796\u001b[0m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 797\u001b[0m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 798\u001b[0m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mretries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 799\u001b[0m \u001b[43m \u001b[49m\u001b[43mresponse_conn\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mresponse_conn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 800\u001b[0m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpreload_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 801\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 802\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mresponse_kw\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 803\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 805\u001b[0m \u001b[38;5;66;03m# Everything went great!\u001b[39;00m\n\u001b[1;32m 806\u001b[0m clean_exit \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n",
|
|
||||||
"File \u001b[0;32m~/.conda/envs/dscb/lib/python3.9/site-packages/urllib3/connectionpool.py:536\u001b[0m, in \u001b[0;36mHTTPConnectionPool._make_request\u001b[0;34m(self, conn, method, url, body, headers, retries, timeout, chunked, response_conn, preload_content, decode_content, enforce_content_length)\u001b[0m\n\u001b[1;32m 534\u001b[0m \u001b[38;5;66;03m# Receive the response from the server\u001b[39;00m\n\u001b[1;32m 535\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 536\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[43mconn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgetresponse\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 537\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (BaseSSLError, \u001b[38;5;167;01mOSError\u001b[39;00m) \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m 538\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_raise_timeout(err\u001b[38;5;241m=\u001b[39me, url\u001b[38;5;241m=\u001b[39murl, timeout_value\u001b[38;5;241m=\u001b[39mread_timeout)\n",
|
|
||||||
"File \u001b[0;32m~/.conda/envs/dscb/lib/python3.9/site-packages/urllib3/connection.py:454\u001b[0m, in \u001b[0;36mHTTPConnection.getresponse\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 451\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mresponse\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m HTTPResponse\n\u001b[1;32m 453\u001b[0m \u001b[38;5;66;03m# Get the response from http.client.HTTPConnection\u001b[39;00m\n\u001b[0;32m--> 454\u001b[0m httplib_response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgetresponse\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 456\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 457\u001b[0m assert_header_parsing(httplib_response\u001b[38;5;241m.\u001b[39mmsg)\n",
|
|
||||||
"File \u001b[0;32m~/.conda/envs/dscb/lib/python3.9/http/client.py:1377\u001b[0m, in \u001b[0;36mHTTPConnection.getresponse\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1375\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1376\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 1377\u001b[0m \u001b[43mresponse\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbegin\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1378\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m:\n\u001b[1;32m 1379\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mclose()\n",
|
|
||||||
"File \u001b[0;32m~/.conda/envs/dscb/lib/python3.9/http/client.py:320\u001b[0m, in \u001b[0;36mHTTPResponse.begin\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 318\u001b[0m \u001b[38;5;66;03m# read until we get a non-100 response\u001b[39;00m\n\u001b[1;32m 319\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[0;32m--> 320\u001b[0m version, status, reason \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_read_status\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 321\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m status \u001b[38;5;241m!=\u001b[39m CONTINUE:\n\u001b[1;32m 322\u001b[0m \u001b[38;5;28;01mbreak\u001b[39;00m\n",
|
|
||||||
"File \u001b[0;32m~/.conda/envs/dscb/lib/python3.9/http/client.py:281\u001b[0m, in \u001b[0;36mHTTPResponse._read_status\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 280\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_read_status\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m--> 281\u001b[0m line \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mstr\u001b[39m(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreadline\u001b[49m\u001b[43m(\u001b[49m\u001b[43m_MAXLINE\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m)\u001b[49m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124miso-8859-1\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 282\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(line) \u001b[38;5;241m>\u001b[39m _MAXLINE:\n\u001b[1;32m 283\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m LineTooLong(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mstatus line\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
|
|
||||||
"File \u001b[0;32m~/.conda/envs/dscb/lib/python3.9/socket.py:704\u001b[0m, in \u001b[0;36mSocketIO.readinto\u001b[0;34m(self, b)\u001b[0m\n\u001b[1;32m 702\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[1;32m 703\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 704\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_sock\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrecv_into\u001b[49m\u001b[43m(\u001b[49m\u001b[43mb\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 705\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m timeout:\n\u001b[1;32m 706\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_timeout_occurred \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n",
|
|
||||||
"\u001b[0;31mKeyboardInterrupt\u001b[0m: "
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"import sseclient\n",
|
|
||||||
"\n",
|
|
||||||
"obj = {\n",
|
|
||||||
" \"inputs\": sequence,\n",
|
|
||||||
" \"generation_parameters\": {\n",
|
|
||||||
" \"max_new_tokens\":100,\n",
|
|
||||||
" # \"repetion_penalty\": 2.0,\n",
|
|
||||||
" # \"do_sample\": False,\n",
|
|
||||||
" # \"temperature\": 1.0,\n",
|
|
||||||
" # \"top_k\": 1,\n",
|
|
||||||
" # \"top_p\": 0.9,\n",
|
|
||||||
" # \"seed\": 43,\n",
|
|
||||||
" }\n",
|
|
||||||
" }\n",
|
|
||||||
"\n",
|
|
||||||
"stream_response = requests.post(url, json=obj, stream=True)\n",
|
|
||||||
"\n",
|
|
||||||
"client = sseclient.SSEClient(stream_response)\n",
|
|
||||||
"\n",
|
|
||||||
"# Loop forever (while connection \"open\")\n",
|
|
||||||
"for event in client.events():\n",
|
|
||||||
" print (\"got a new event from server\")\n",
|
|
||||||
" pprint.pprint(event.data)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 95,
|
|
||||||
"id": "9cac401e-3a69-4f9c-b074-f1ba8e06df49",
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"Collecting sseclient\n",
|
|
||||||
" Downloading sseclient-0.0.27.tar.gz (7.5 kB)\n",
|
|
||||||
" Preparing metadata (setup.py) ... \u001b[?25ldone\n",
|
|
||||||
"\u001b[?25hRequirement already satisfied: requests>=2.9 in /home/robertgshaw/.conda/envs/dscb/lib/python3.9/site-packages (from sseclient) (2.31.0)\n",
|
|
||||||
"Requirement already satisfied: six in /home/robertgshaw/.conda/envs/dscb/lib/python3.9/site-packages (from sseclient) (1.16.0)\n",
|
|
||||||
"Requirement already satisfied: charset-normalizer<4,>=2 in /home/robertgshaw/.conda/envs/dscb/lib/python3.9/site-packages (from requests>=2.9->sseclient) (3.1.0)\n",
|
|
||||||
"Requirement already satisfied: idna<4,>=2.5 in /home/robertgshaw/.conda/envs/dscb/lib/python3.9/site-packages (from requests>=2.9->sseclient) (3.4)\n",
|
|
||||||
"Requirement already satisfied: urllib3<3,>=1.21.1 in /home/robertgshaw/.conda/envs/dscb/lib/python3.9/site-packages (from requests>=2.9->sseclient) (2.0.3)\n",
|
|
||||||
"Requirement already satisfied: certifi>=2017.4.17 in /home/robertgshaw/.conda/envs/dscb/lib/python3.9/site-packages (from requests>=2.9->sseclient) (2023.5.7)\n",
|
|
||||||
"Building wheels for collected packages: sseclient\n",
|
|
||||||
" Building wheel for sseclient (setup.py) ... \u001b[?25ldone\n",
|
|
||||||
"\u001b[?25h Created wheel for sseclient: filename=sseclient-0.0.27-py3-none-any.whl size=5565 sha256=56a8fb4039994d2facce58dba0bad7a3af7f5a9569e80b70ec2adab94dc46a35\n",
|
|
||||||
" Stored in directory: /home/robertgshaw/.cache/pip/wheels/73/21/3f/9d20c05c1c9312ab61bc7bf63bb2c189a71e800cb028fa8d2a\n",
|
|
||||||
"Successfully built sseclient\n",
|
|
||||||
"Installing collected packages: sseclient\n",
|
|
||||||
"Successfully installed sseclient-0.0.27\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"!pip install sseclient"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 96,
|
|
||||||
"id": "80272ae9-6c07-43a3-920e-4d13a5905685",
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"ename": "InvalidURL",
|
|
||||||
"evalue": "Failed to parse: <Response [405]>",
|
|
||||||
"output_type": "error",
|
|
||||||
"traceback": [
|
|
||||||
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
|
||||||
"\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)",
|
|
||||||
"File \u001b[0;32m~/.conda/envs/dscb/lib/python3.9/site-packages/urllib3/util/url.py:425\u001b[0m, in \u001b[0;36mparse_url\u001b[0;34m(url)\u001b[0m\n\u001b[1;32m 424\u001b[0m auth \u001b[38;5;241m=\u001b[39m auth \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m--> 425\u001b[0m host, port \u001b[38;5;241m=\u001b[39m \u001b[43m_HOST_PORT_RE\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmatch\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhost_port\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgroups\u001b[49m() \u001b[38;5;66;03m# type: ignore[union-attr]\u001b[39;00m\n\u001b[1;32m 426\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m auth \u001b[38;5;129;01mand\u001b[39;00m normalize_uri:\n",
|
|
||||||
"\u001b[0;31mAttributeError\u001b[0m: 'NoneType' object has no attribute 'groups'",
|
|
||||||
"\nThe above exception was the direct cause of the following exception:\n",
|
|
||||||
"\u001b[0;31mLocationParseError\u001b[0m Traceback (most recent call last)",
|
|
||||||
"File \u001b[0;32m~/.conda/envs/dscb/lib/python3.9/site-packages/requests/models.py:434\u001b[0m, in \u001b[0;36mPreparedRequest.prepare_url\u001b[0;34m(self, url, params)\u001b[0m\n\u001b[1;32m 433\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 434\u001b[0m scheme, auth, host, port, path, query, fragment \u001b[38;5;241m=\u001b[39m \u001b[43mparse_url\u001b[49m\u001b[43m(\u001b[49m\u001b[43murl\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 435\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m LocationParseError \u001b[38;5;28;01mas\u001b[39;00m e:\n",
|
|
||||||
"File \u001b[0;32m~/.conda/envs/dscb/lib/python3.9/site-packages/urllib3/util/url.py:451\u001b[0m, in \u001b[0;36mparse_url\u001b[0;34m(url)\u001b[0m\n\u001b[1;32m 450\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (\u001b[38;5;167;01mValueError\u001b[39;00m, \u001b[38;5;167;01mAttributeError\u001b[39;00m) \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m--> 451\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m LocationParseError(source_url) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01me\u001b[39;00m\n\u001b[1;32m 453\u001b[0m \u001b[38;5;66;03m# For the sake of backwards compatibility we put empty\u001b[39;00m\n\u001b[1;32m 454\u001b[0m \u001b[38;5;66;03m# string values for path if there are any defined values\u001b[39;00m\n\u001b[1;32m 455\u001b[0m \u001b[38;5;66;03m# beyond the path in the URL.\u001b[39;00m\n\u001b[1;32m 456\u001b[0m \u001b[38;5;66;03m# TODO: Remove this when we break backwards compatibility.\u001b[39;00m\n",
|
|
||||||
"\u001b[0;31mLocationParseError\u001b[0m: Failed to parse: <Response [405]>",
|
|
||||||
"\nDuring handling of the above exception, another exception occurred:\n",
|
|
||||||
"\u001b[0;31mInvalidURL\u001b[0m Traceback (most recent call last)",
|
|
||||||
"Cell \u001b[0;32mIn[96], line 4\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01msseclient\u001b[39;00m\u001b[38;5;241m,\u001b[39m \u001b[38;5;21;01mpprint\u001b[39;00m\n\u001b[1;32m 3\u001b[0m stream_response \u001b[38;5;241m=\u001b[39m requests\u001b[38;5;241m.\u001b[39mget(url, json\u001b[38;5;241m=\u001b[39mobj, stream\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[0;32m----> 4\u001b[0m client \u001b[38;5;241m=\u001b[39m \u001b[43msseclient\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mSSEClient\u001b[49m\u001b[43m(\u001b[49m\u001b[43mstream_response\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 6\u001b[0m \u001b[38;5;66;03m# Loop forever (while connection \"open\")\u001b[39;00m\n\u001b[1;32m 7\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m event \u001b[38;5;129;01min\u001b[39;00m client\u001b[38;5;241m.\u001b[39mevents():\n",
|
|
||||||
"File \u001b[0;32m~/.conda/envs/dscb/lib/python3.9/site-packages/sseclient.py:48\u001b[0m, in \u001b[0;36mSSEClient.__init__\u001b[0;34m(self, url, last_id, retry, session, chunk_size, **kwargs)\u001b[0m\n\u001b[1;32m 45\u001b[0m \u001b[38;5;66;03m# Keep data here as it streams in\u001b[39;00m\n\u001b[1;32m 46\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbuf \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[0;32m---> 48\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_connect\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
|
|
||||||
"File \u001b[0;32m~/.conda/envs/dscb/lib/python3.9/site-packages/sseclient.py:56\u001b[0m, in \u001b[0;36mSSEClient._connect\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 54\u001b[0m \u001b[38;5;66;03m# Use session if set. Otherwise fall back to requests module.\u001b[39;00m\n\u001b[1;32m 55\u001b[0m requester \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msession \u001b[38;5;129;01mor\u001b[39;00m requests\n\u001b[0;32m---> 56\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mresp \u001b[38;5;241m=\u001b[39m \u001b[43mrequester\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstream\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequests_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 57\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mresp_iterator \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39miter_content()\n\u001b[1;32m 58\u001b[0m encoding \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mresp\u001b[38;5;241m.\u001b[39mencoding \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mresp\u001b[38;5;241m.\u001b[39mapparent_encoding\n",
|
|
||||||
"File \u001b[0;32m~/.conda/envs/dscb/lib/python3.9/site-packages/requests/api.py:73\u001b[0m, in \u001b[0;36mget\u001b[0;34m(url, params, **kwargs)\u001b[0m\n\u001b[1;32m 62\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mget\u001b[39m(url, params\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 63\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124mr\u001b[39m\u001b[38;5;124;03m\"\"\"Sends a GET request.\u001b[39;00m\n\u001b[1;32m 64\u001b[0m \n\u001b[1;32m 65\u001b[0m \u001b[38;5;124;03m :param url: URL for the new :class:`Request` object.\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 70\u001b[0m \u001b[38;5;124;03m :rtype: requests.Response\u001b[39;00m\n\u001b[1;32m 71\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m---> 73\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mget\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mparams\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mparams\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
|
|
||||||
"File \u001b[0;32m~/.conda/envs/dscb/lib/python3.9/site-packages/requests/api.py:59\u001b[0m, in \u001b[0;36mrequest\u001b[0;34m(method, url, **kwargs)\u001b[0m\n\u001b[1;32m 55\u001b[0m \u001b[38;5;66;03m# By using the 'with' statement we are sure the session is closed, thus we\u001b[39;00m\n\u001b[1;32m 56\u001b[0m \u001b[38;5;66;03m# avoid leaving sockets open which can trigger a ResourceWarning in some\u001b[39;00m\n\u001b[1;32m 57\u001b[0m \u001b[38;5;66;03m# cases, and look like a memory leak in others.\u001b[39;00m\n\u001b[1;32m 58\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m sessions\u001b[38;5;241m.\u001b[39mSession() \u001b[38;5;28;01mas\u001b[39;00m session:\n\u001b[0;32m---> 59\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43msession\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmethod\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
|
|
||||||
"File \u001b[0;32m~/.conda/envs/dscb/lib/python3.9/site-packages/requests/sessions.py:575\u001b[0m, in \u001b[0;36mSession.request\u001b[0;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[0m\n\u001b[1;32m 562\u001b[0m \u001b[38;5;66;03m# Create the Request.\u001b[39;00m\n\u001b[1;32m 563\u001b[0m req \u001b[38;5;241m=\u001b[39m Request(\n\u001b[1;32m 564\u001b[0m method\u001b[38;5;241m=\u001b[39mmethod\u001b[38;5;241m.\u001b[39mupper(),\n\u001b[1;32m 565\u001b[0m url\u001b[38;5;241m=\u001b[39murl,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 573\u001b[0m hooks\u001b[38;5;241m=\u001b[39mhooks,\n\u001b[1;32m 574\u001b[0m )\n\u001b[0;32m--> 575\u001b[0m prep \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mprepare_request\u001b[49m\u001b[43m(\u001b[49m\u001b[43mreq\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 577\u001b[0m proxies \u001b[38;5;241m=\u001b[39m proxies \u001b[38;5;129;01mor\u001b[39;00m {}\n\u001b[1;32m 579\u001b[0m settings \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmerge_environment_settings(\n\u001b[1;32m 580\u001b[0m prep\u001b[38;5;241m.\u001b[39murl, proxies, stream, verify, cert\n\u001b[1;32m 581\u001b[0m )\n",
|
|
||||||
"File \u001b[0;32m~/.conda/envs/dscb/lib/python3.9/site-packages/requests/sessions.py:486\u001b[0m, in \u001b[0;36mSession.prepare_request\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m 483\u001b[0m auth \u001b[38;5;241m=\u001b[39m get_netrc_auth(request\u001b[38;5;241m.\u001b[39murl)\n\u001b[1;32m 485\u001b[0m p \u001b[38;5;241m=\u001b[39m PreparedRequest()\n\u001b[0;32m--> 486\u001b[0m \u001b[43mp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mprepare\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 487\u001b[0m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmethod\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mupper\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 488\u001b[0m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 489\u001b[0m \u001b[43m \u001b[49m\u001b[43mfiles\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfiles\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 490\u001b[0m \u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 491\u001b[0m \u001b[43m \u001b[49m\u001b[43mjson\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mjson\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 492\u001b[0m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmerge_setting\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 493\u001b[0m \u001b[43m \u001b[49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdict_class\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mCaseInsensitiveDict\u001b[49m\n\u001b[1;32m 494\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 495\u001b[0m \u001b[43m \u001b[49m\u001b[43mparams\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmerge_setting\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mparams\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mparams\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 496\u001b[0m \u001b[43m \u001b[49m\u001b[43mauth\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmerge_setting\u001b[49m\u001b[43m(\u001b[49m\u001b[43mauth\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mauth\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 497\u001b[0m \u001b[43m \u001b[49m\u001b[43mcookies\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmerged_cookies\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 498\u001b[0m \u001b[43m \u001b[49m\u001b[43mhooks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmerge_hooks\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mhooks\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mhooks\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 499\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 500\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m p\n",
|
|
||||||
"File \u001b[0;32m~/.conda/envs/dscb/lib/python3.9/site-packages/requests/models.py:368\u001b[0m, in \u001b[0;36mPreparedRequest.prepare\u001b[0;34m(self, method, url, headers, files, data, params, auth, cookies, hooks, json)\u001b[0m\n\u001b[1;32m 365\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Prepares the entire request with the given parameters.\"\"\"\u001b[39;00m\n\u001b[1;32m 367\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprepare_method(method)\n\u001b[0;32m--> 368\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mprepare_url\u001b[49m\u001b[43m(\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mparams\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 369\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprepare_headers(headers)\n\u001b[1;32m 370\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprepare_cookies(cookies)\n",
|
|
||||||
"File \u001b[0;32m~/.conda/envs/dscb/lib/python3.9/site-packages/requests/models.py:436\u001b[0m, in \u001b[0;36mPreparedRequest.prepare_url\u001b[0;34m(self, url, params)\u001b[0m\n\u001b[1;32m 434\u001b[0m scheme, auth, host, port, path, query, fragment \u001b[38;5;241m=\u001b[39m parse_url(url)\n\u001b[1;32m 435\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m LocationParseError \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m--> 436\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m InvalidURL(\u001b[38;5;241m*\u001b[39me\u001b[38;5;241m.\u001b[39margs)\n\u001b[1;32m 438\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m scheme:\n\u001b[1;32m 439\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m MissingSchema(\n\u001b[1;32m 440\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mInvalid URL \u001b[39m\u001b[38;5;132;01m{\u001b[39;00murl\u001b[38;5;132;01m!r}\u001b[39;00m\u001b[38;5;124m: No scheme supplied. \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 441\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mPerhaps you meant https://\u001b[39m\u001b[38;5;132;01m{\u001b[39;00murl\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m?\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 442\u001b[0m )\n",
|
|
||||||
"\u001b[0;31mInvalidURL\u001b[0m: Failed to parse: <Response [405]>"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"import sseclient, pprint\n",
|
|
||||||
"\n",
|
|
||||||
"stream_response = requests.get(url, json=obj, stream=True)\n",
|
|
||||||
"client = sseclient.SSEClient(stream_response)\n",
|
|
||||||
"\n",
|
|
||||||
"# Loop forever (while connection \"open\")\n",
|
|
||||||
"for event in client.events():\n",
|
|
||||||
" print (\"got a new event from server\")\n",
|
|
||||||
" pprint.pprint(event.data)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 93,
|
|
||||||
"id": "7513b2a1-0749-44b5-88a9-d91d5b175e8e",
|
"id": "7513b2a1-0749-44b5-88a9-d91d5b175e8e",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [
|
"outputs": [
|
||||||
{
|
|
||||||
"name": "stderr",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"Exception in thread Thread-179:\n",
|
|
||||||
"Traceback (most recent call last):\n",
|
|
||||||
" File \"/home/robertgshaw/.conda/envs/dscb/lib/python3.9/threading.py\", line 980, in _bootstrap_inner\n",
|
|
||||||
" self.run()\n",
|
|
||||||
" File \"/home/robertgshaw/.conda/envs/dscb/lib/python3.9/threading.py\", line 917, in run\n",
|
|
||||||
" self._target(*self._args, **self._kwargs)\n",
|
|
||||||
" File \"/tmp/ipykernel_9240/1133841396.py\", line 26, in request_task\n",
|
|
||||||
"TypeError: 'NoneType' object is not subscriptable\n"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"name": "stdout",
|
"name": "stdout",
|
||||||
"output_type": "stream",
|
"output_type": "stream",
|
||||||
"text": [
|
"text": [
|
||||||
"55\n",
|
"25\n",
|
||||||
"None\n"
|
"Write a function for computing a fibonacci sequence: \n",
|
||||||
|
"\n",
|
||||||
|
"def fib(n):\n",
|
||||||
|
" if n == 1:\n",
|
||||||
|
" return 0\n",
|
||||||
|
" elif n == 2:\n",
|
||||||
|
" return 1\n",
|
||||||
|
"\n",
|
||||||
|
"50\n",
|
||||||
|
"Write a function for computing a fibonacci sequence: \n",
|
||||||
|
"\n",
|
||||||
|
"def fib(n):\n",
|
||||||
|
" if n == 1:\n",
|
||||||
|
" return 0\n",
|
||||||
|
" elif n == 2:\n",
|
||||||
|
" return 1\n",
|
||||||
|
" else:\n",
|
||||||
|
" return fib(n - 1) + fib(n - 2)\n",
|
||||||
|
"\n",
|
||||||
|
"def fib_seq\n",
|
||||||
|
"100\n",
|
||||||
|
"Write a function for computing a fibonacci sequence: \n",
|
||||||
|
"\n",
|
||||||
|
"def fib(n):\n",
|
||||||
|
" if n == 1:\n",
|
||||||
|
" return 0\n",
|
||||||
|
" elif n == 2:\n",
|
||||||
|
" return 1\n",
|
||||||
|
" else:\n",
|
||||||
|
" return fib(n - 1) + fib(n - 2)\n",
|
||||||
|
"\n",
|
||||||
|
"def fib_seq_1(n):\n",
|
||||||
|
" if n == 1:\n",
|
||||||
|
" return 0\n",
|
||||||
|
" elif n == 2:\n",
|
||||||
|
" return 1\n",
|
||||||
|
" else:\n",
|
||||||
|
" return fib_seq_1(n - 1) + fib_seq_\n"
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
@ -202,11 +57,11 @@
|
|||||||
"from threading import Thread\n",
|
"from threading import Thread\n",
|
||||||
"import json\n",
|
"import json\n",
|
||||||
"\n",
|
"\n",
|
||||||
"url = \"http://127.0.0.1:5543/generate_stream\"\n",
|
"url = \"http://127.0.0.1:5543/generate\"\n",
|
||||||
"sequence = \"Finish the following function for computing a fibonacci sequence: \\n\\ndef fib(n):\"\n",
|
"sequence = \"Write a function for computing a fibonacci sequence: \\n\\ndef fib(n):\"\n",
|
||||||
"# sequence = \"def fib(n):\"\n",
|
"# sequence = \"def fib(n):\"\n",
|
||||||
"\n",
|
"\n",
|
||||||
"def request_task(seed, max_new_tokens):\n",
|
"def request_task(max_new_tokens):\n",
|
||||||
" obj = {\n",
|
" obj = {\n",
|
||||||
" \"inputs\": sequence,\n",
|
" \"inputs\": sequence,\n",
|
||||||
" \"generation_parameters\": {\n",
|
" \"generation_parameters\": {\n",
|
||||||
@ -216,23 +71,19 @@
|
|||||||
" # \"temperature\": 1.1,\n",
|
" # \"temperature\": 1.1,\n",
|
||||||
" # \"top_k\": 3,\n",
|
" # \"top_k\": 3,\n",
|
||||||
" # \"top_p\": 0.9,\n",
|
" # \"top_p\": 0.9,\n",
|
||||||
" \"seed\": seed,\n",
|
" # \"seed\": 42,\n",
|
||||||
" }\n",
|
" }\n",
|
||||||
" }\n",
|
" }\n",
|
||||||
" with requests.post(url, json=obj) as r:\n",
|
" with requests.post(url, json=obj) as r:\n",
|
||||||
" print(max_new_tokens)\n",
|
" print(max_new_tokens)\n",
|
||||||
" dct = json.loads(r.text)\n",
|
" dct = json.loads(r.text)\n",
|
||||||
" print(dct)\n",
|
" # print(dct)\n",
|
||||||
" print(f'{sequence}{dct[\"response_text\"]}')\n",
|
" print(f'{sequence}{dct[\"response_text\"]}')\n",
|
||||||
"\n",
|
"\n",
|
||||||
"max_new_tokens_lst = [55, 50, 45]\n",
|
"max_new_tokens_lst = [100, 50, 25]\n",
|
||||||
"seeds = [1,2,3]\n",
|
|
||||||
"# max_new_tokens_lst = [100, 200, 300]\n",
|
|
||||||
"\n",
|
|
||||||
"cnt = 1\n",
|
|
||||||
"request_ts = [\n",
|
"request_ts = [\n",
|
||||||
" Thread(target=request_task, args=[seed, max_new_tokens]) \n",
|
" Thread(target=request_task, args=[max_new_tokens]) \n",
|
||||||
" for seed, max_new_tokens in zip(seeds[:cnt], max_new_tokens_lst[:cnt])\n",
|
" for max_new_tokens in max_new_tokens_lst\n",
|
||||||
"]\n",
|
"]\n",
|
||||||
"\n",
|
"\n",
|
||||||
"import time\n",
|
"import time\n",
|
||||||
@ -2755,7 +2606,7 @@
|
|||||||
"name": "python",
|
"name": "python",
|
||||||
"nbconvert_exporter": "python",
|
"nbconvert_exporter": "python",
|
||||||
"pygments_lexer": "ipython3",
|
"pygments_lexer": "ipython3",
|
||||||
"version": "3.9.17"
|
"version": "3.9.2"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"nbformat": 4,
|
"nbformat": 4,
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user