huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-10 20:04:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

revert

Browse Source
This commit is contained in:
OlivierDehaene 2023-07-19 01:15:59 +02:00
parent 8793ae5890
commit 7f399cd848
1 changed files with 2 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
server/text_generation_server/server.py
View File

@ -69,8 +69,6 @@ class TextGenerationService(generate_pb2_grpc.TextGenerationServiceServicer):
) )
generations, next_batch = self.model.generate_token(batch) generations, next_batch = self.model.generate_token(batch)
if next_batch is not None:
self.cache.set(next_batch) self.cache.set(next_batch)
return generate_pb2.PrefillResponse( return generate_pb2.PrefillResponse(
@ -98,8 +96,6 @@ class TextGenerationService(generate_pb2_grpc.TextGenerationServiceServicer):
batch = batches[0] batch = batches[0]
generations, next_batch = self.model.generate_token(batch) generations, next_batch = self.model.generate_token(batch)
if next_batch is not None:
self.cache.set(next_batch) self.cache.set(next_batch)
return generate_pb2.DecodeResponse( return generate_pb2.DecodeResponse(