huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-10 11:54:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

revert

Browse Source
This commit is contained in:
OlivierDehaene 2023-07-19 01:15:59 +02:00
parent 8793ae5890
commit 7f399cd848
1 changed files with 2 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
server/text_generation_server/server.py
View File

@ -69,9 +69,7 @@ class TextGenerationService(generate_pb2_grpc.TextGenerationServiceServicer):
)
generations, next_batch = self.model.generate_token(batch)
if next_batch is not None:
self.cache.set(next_batch)
self.cache.set(next_batch)
return generate_pb2.PrefillResponse(
generations=[generation.to_pb() for generation in generations],
@ -98,9 +96,7 @@ class TextGenerationService(generate_pb2_grpc.TextGenerationServiceServicer):
batch = batches[0]
generations, next_batch = self.model.generate_token(batch)
if next_batch is not None:
self.cache.set(next_batch)
self.cache.set(next_batch)
return generate_pb2.DecodeResponse(
generations=[generation.to_pb() for generation in generations],