huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-12 04:44:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Integrations tests for mllama (cutting to 10 tokens because there seems'

Browse Source 
to be instability after (meaning size of the batch matters.
This commit is contained in:
Nicolas Patry 2024-09-28 22:41:07 +02:00
parent 2ac607a215
commit af677caf4f
No known key found for this signature in database
GPG Key ID: 64AF4752B2967863
3 changed files with 43 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
integration-tests/models/__snapshots__/test_mllama/test_mllama_load.json
View File

@ -6,7 +6,7 @@
"index": 0, "index": 0,
"logprobs": null, "logprobs": null,
"message": { "message": {
"content": "In a small village, a rooster named Cluck Norris ruled the coop with an iron beak", "content": "In a bustling city, a chicken named Cluck",
"name": null, "name": null,
"role": "assistant", "role": "assistant",
"tool_calls": null "tool_calls": null
@ -14,15 +14,15 @@
"usage": null "usage": null
} }
], ],
"created": 1727097740, "created": 1727555830,
"id": "", "id": "",
"model": "s0409/model-3", "model": "meta-llama/Llama-3.2-11B-Vision-Instruct",
"object": "chat.completion", "object": "chat.completion",
"system_fingerprint": "2.2.1-dev0-native", "system_fingerprint": "2.3.1-dev0-native",
"usage": { "usage": {
"completion_tokens": 20, "completion_tokens": 10,
"prompt_tokens": 24, "prompt_tokens": 50,
"total_tokens": 44 "total_tokens": 60
} }
}, },
{ {
@ -32,7 +32,7 @@
"index": 0, "index": 0,
"logprobs": null, "logprobs": null,
"message": { "message": {
"content": "In a small village, a rooster named Cluck Norris ruled the coop with an iron beak", "content": "In a bustling city, a chicken named Cluck",
"name": null, "name": null,
"role": "assistant", "role": "assistant",
"tool_calls": null "tool_calls": null
@ -40,15 +40,15 @@
"usage": null "usage": null
} }
], ],
"created": 1727097740, "created": 1727555830,
"id": "", "id": "",
"model": "s0409/model-3", "model": "meta-llama/Llama-3.2-11B-Vision-Instruct",
"object": "chat.completion", "object": "chat.completion",
"system_fingerprint": "2.2.1-dev0-native", "system_fingerprint": "2.3.1-dev0-native",
"usage": { "usage": {
"completion_tokens": 20, "completion_tokens": 10,
"prompt_tokens": 24, "prompt_tokens": 50,
"total_tokens": 44 "total_tokens": 60
} }
}, },
{ {
@ -58,7 +58,7 @@
"index": 0, "index": 0,
"logprobs": null, "logprobs": null,
"message": { "message": {
"content": "In a small village, a rooster named Cluck Norris ruled the coop with an iron beak", "content": "In a bustling city, a chicken named Cluck",
"name": null, "name": null,
"role": "assistant", "role": "assistant",
"tool_calls": null "tool_calls": null
@ -66,15 +66,15 @@
"usage": null "usage": null
} }
], ],
"created": 1727097740, "created": 1727555830,
"id": "", "id": "",
"model": "s0409/model-3", "model": "meta-llama/Llama-3.2-11B-Vision-Instruct",
"object": "chat.completion", "object": "chat.completion",
"system_fingerprint": "2.2.1-dev0-native", "system_fingerprint": "2.3.1-dev0-native",
"usage": { "usage": {
"completion_tokens": 20, "completion_tokens": 10,
"prompt_tokens": 24, "prompt_tokens": 50,
"total_tokens": 44 "total_tokens": 60
} }
}, },
{ {
@ -84,7 +84,7 @@
"index": 0, "index": 0,
"logprobs": null, "logprobs": null,
"message": { "message": {
"content": "In a small village, a rooster named Cluck Norris ruled the coop with an iron beak", "content": "In a bustling city, a chicken named Cluck",
"name": null, "name": null,
"role": "assistant", "role": "assistant",
"tool_calls": null "tool_calls": null
@ -92,15 +92,15 @@
"usage": null "usage": null
} }
], ],
"created": 1727097740, "created": 1727555830,
"id": "", "id": "",
"model": "s0409/model-3", "model": "meta-llama/Llama-3.2-11B-Vision-Instruct",
"object": "chat.completion", "object": "chat.completion",
"system_fingerprint": "2.2.1-dev0-native", "system_fingerprint": "2.3.1-dev0-native",
"usage": { "usage": {
"completion_tokens": 20, "completion_tokens": 10,
"prompt_tokens": 24, "prompt_tokens": 50,
"total_tokens": 44 "total_tokens": 60
} }
} }
] ]

14
integration-tests/models/__snapshots__/test_mllama/test_mllama_simpl.json
View File

@ -5,7 +5,7 @@
"index": 0, "index": 0,
"logprobs": null, "logprobs": null,
"message": { "message": {
"content": "In a small village, a rooster named Cluck Norris ruled the coop with an iron beak", "content": "In a bustling city, a chicken named Cluck",
"name": null, "name": null,
"role": "assistant", "role": "assistant",
"tool_calls": null "tool_calls": null
@ -13,14 +13,14 @@
"usage": null "usage": null
} }
], ],
"created": 1727090615, "created": 1727556016,
"id": "", "id": "",
"model": "s0409/model-3", "model": "meta-llama/Llama-3.2-11B-Vision-Instruct",
"object": "chat.completion", "object": "chat.completion",
"system_fingerprint": "2.2.1-dev0-native", "system_fingerprint": "2.3.1-dev0-native",
"usage": { "usage": {
"completion_tokens": 20, "completion_tokens": 10,
"prompt_tokens": 24, "prompt_tokens": 50,
"total_tokens": 44 "total_tokens": 60
} }
} }

19
integration-tests/models/test_mllama.py
View File

@ -5,7 +5,7 @@ import asyncio
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def mllama_handle(launcher): def mllama_handle(launcher):
with launcher("s0409/model-3", num_shard=2) as handle: with launcher("meta-llama/Llama-3.2-11B-Vision-Instruct", num_shard=2) as handle:
yield handle yield handle
@ -32,7 +32,7 @@ def get_cow_beach():
async def test_mllama_simpl(mllama, response_snapshot): async def test_mllama_simpl(mllama, response_snapshot):
# chicken = get_chicken() # chicken = get_chicken()
response = await mllama.chat( response = await mllama.chat(
max_tokens=20, max_tokens=10,
temperature=0.0, temperature=0.0,
messages=[ messages=[
{ {
@ -54,13 +54,13 @@ async def test_mllama_simpl(mllama, response_snapshot):
) )
assert response.usage == { assert response.usage == {
"completion_tokens": 20, "completion_tokens": 10,
"prompt_tokens": 24, "prompt_tokens": 50,
"total_tokens": 44, "total_tokens": 60,
} }
assert ( assert (
response.choices[0].message.content response.choices[0].message.content
== "In a small village, a rooster named Cluck Norris ruled the coop with an iron beak" == "In a bustling city, a chicken named Cluck"
) )
assert response == response_snapshot assert response == response_snapshot
@ -70,7 +70,7 @@ async def test_mllama_simpl(mllama, response_snapshot):
async def test_mllama_load(mllama, generate_load, response_snapshot): async def test_mllama_load(mllama, generate_load, response_snapshot):
futures = [ futures = [
mllama.chat( mllama.chat(
max_tokens=20, max_tokens=10,
temperature=0.0, temperature=0.0,
messages=[ messages=[
{ {
@ -96,10 +96,7 @@ async def test_mllama_load(mllama, generate_load, response_snapshot):
generated_texts = [response.choices[0].message.content for response in responses] generated_texts = [response.choices[0].message.content for response in responses]
assert ( assert generated_texts[0] == "In a bustling city, a chicken named Cluck"
generated_texts[0]
== "In a small village, a rooster named Cluck Norris ruled the coop with an iron beak"
)
assert len(generated_texts) == 4 assert len(generated_texts) == 4
assert generated_texts, all( assert generated_texts, all(
[text == generated_texts[0] for text in generated_texts] [text == generated_texts[0] for text in generated_texts]