huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-12 04:44:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Adding a test for FD.

Browse Source
This commit is contained in:
Nicolas Patry 2024-09-12 11:12:18 +02:00
parent 9cca3e0b03
commit f6697baf31
No known key found for this signature in database
GPG Key ID: 64AF4752B2967863
3 changed files with 2784 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
integration-tests/conftest.py
View File

@ -342,6 +342,7 @@ def launcher(event_loop):
max_total_tokens: Optional[int] = None, max_total_tokens: Optional[int] = None,
lora_adapters: Optional[List[str]] = None, lora_adapters: Optional[List[str]] = None,
cuda_graphs: Optional[List[int]] = None, cuda_graphs: Optional[List[int]] = None,
attention: Optional[str] = None,
): ):
port = random.randint(8000, 10_000) port = random.randint(8000, 10_000)
master_port = random.randint(10_000, 20_000) master_port = random.randint(10_000, 20_000)
@ -401,6 +402,8 @@ def launcher(event_loop):
if not use_flash_attention: if not use_flash_attention:
env["USE_FLASH_ATTENTION"] = "false" env["USE_FLASH_ATTENTION"] = "false"
if attention is not None:
env["ATTENTION"] = attention
with tempfile.TemporaryFile("w+") as tmp: with tempfile.TemporaryFile("w+") as tmp:
# We'll output stdout/stderr to a temporary file. Using a pipe # We'll output stdout/stderr to a temporary file. Using a pipe
@ -437,6 +440,7 @@ def launcher(event_loop):
max_total_tokens: Optional[int] = None, max_total_tokens: Optional[int] = None,
lora_adapters: Optional[List[str]] = None, lora_adapters: Optional[List[str]] = None,
cuda_graphs: Optional[List[int]] = None, cuda_graphs: Optional[List[int]] = None,
attention: Optional[str] = None,
): ):
port = random.randint(8000, 10_000) port = random.randint(8000, 10_000)
@ -491,6 +495,8 @@ def launcher(event_loop):
} }
if not use_flash_attention: if not use_flash_attention:
env["USE_FLASH_ATTENTION"] = "false" env["USE_FLASH_ATTENTION"] = "false"
if attention is not None:
env["ATTENTION"] = attention
if HF_TOKEN is not None: if HF_TOKEN is not None:
env["HF_TOKEN"] = HF_TOKEN env["HF_TOKEN"] = HF_TOKEN

2550
integration-tests/models/__snapshots__/test_flash_llama_prefix/test_flash_llama_flashdecoding.json Normal file
View File

File diff suppressed because it is too large Load Diff

228
integration-tests/models/test_flash_llama_prefix.py
View File

File diff suppressed because one or more lines are too long