diff --git a/server/text_generation_server/layers/attention/cuda.py b/server/text_generation_server/layers/attention/cuda.py index a564237c..d87d9199 100644 --- a/server/text_generation_server/layers/attention/cuda.py +++ b/server/text_generation_server/layers/attention/cuda.py @@ -159,6 +159,7 @@ def paged_attention( "auto", 1.0, ) + return out try: diff --git a/server/text_generation_server/layers/attention/rocm.py b/server/text_generation_server/layers/attention/rocm.py index c5c485de..36db12d0 100644 --- a/server/text_generation_server/layers/attention/rocm.py +++ b/server/text_generation_server/layers/attention/rocm.py @@ -129,6 +129,7 @@ def paged_attention( "auto", 1.0, ) + return out if ENGINE != "triton":