From bd5fcf6f133b9d9d46a4f0ba14ee77147dc6414d Mon Sep 17 00:00:00 2001
From: Robert Shaw <114415538+rsnm2@users.noreply.github.com>
Date: Mon, 28 Aug 2023 11:03:54 -0400
Subject: [PATCH] Update server.py

---
 deepsparse/server.py | 35 +----------------------------------
 1 file changed, 1 insertion(+), 34 deletions(-)

diff --git a/deepsparse/server.py b/deepsparse/server.py
index 069edabf..fd9db19c 100644
--- a/deepsparse/server.py
+++ b/deepsparse/server.py
@@ -11,7 +11,6 @@ from utils import GenerateRequestInputs, GenerateRequestOutputs, GenerateRequest
 
 TOKENIZER_PATH = "/home/robertgshaw/.cache/sparsezoo/neuralmagic/codegen_mono-350m-bigpython_bigquery_thepile-base/deployment"
 MODEL_PATH = "/home/robertgshaw/.cache/sparsezoo/neuralmagic/codegen_mono-350m-bigpython_bigquery_thepile-base/model.onnx/model.onnx"
-MESSAGE_STREAM_RETRY_TIMEOUT = 15000  # milisecond
 
 artifacts = {}
 
@@ -52,38 +51,6 @@ def generate(inputs: GenerateRequestInputs) -> GenerateRequestOutputs:
     gr_outputs.finish_reason = generation.finish_reason
     return gr_outputs
 
-@app.post("/generate_stream")
-async def generate_stream(request: fastapi.Request, inputs: GenerateRequestInputs):
-    
-    # convert input to generate request
-    generate_request = GenerateRequest.from_gr_inputs(inputs)
-    
-    # submit request to the router
-    artifacts["router"].submit_request(generate_request)
-
-    async def token_generator(): 
-        while True:
-            if await request.is_disconnected():
-                break
-                
-            generation = generate_request.response_stream.get()
-            if not generation.stopped:
-                yield {
-                    "event": "token_generated",
-                    "id": "message_id",
-                    "retry": MESSAGE_STREAM_RETRY_TIMEOUT,
-                    "data": generation.token
-                }
-            else:
-                yield {
-                    "event": "token_generated",
-                    "id": "message_id",
-                    "retry": MESSAGE_STREAM_RETRY_TIMEOUT,
-                    "data": generation.finish_reason
-                }
-
-    return EventSourceResponse(token_generator())
-
 if __name__ == "__main__":
     uvicorn.run(
         "server:app",
@@ -91,4 +58,4 @@ if __name__ == "__main__":
         port=5543,
         workers=1,      # limit to one process to avoid copying the model
         # reload=True
-    )
\ No newline at end of file
+    )