huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-22 15:32:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update the flaky mllama test.

Browse Source
This commit is contained in:
Nicolas Patry 2025-02-11 17:10:36 +01:00
parent 571ac9b507
commit 8a870b31b9
No known key found for this signature in database
GPG Key ID: 4242CEF24CB6DBF9
3 changed files with 19 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
integration-tests/models/__snapshots__/test_mllama/test_mllama_load.json
View File

@ -6,7 +6,7 @@
"index": 0, "index": 0,
"logprobs": null, "logprobs": null,
"message": { "message": {
"content": "In a small town, a chicken named Cluck", "content": "A chicken sits on a pile of money, looking",
"name": null, "name": null,
"role": "assistant", "role": "assistant",
"tool_calls": null "tool_calls": null
@ -14,15 +14,15 @@
"usage": null "usage": null
} }
], ],
"created": 1738753835, "created": 1739290197,
"id": "", "id": "",
"model": "meta-llama/Llama-3.2-11B-Vision-Instruct", "model": "meta-llama/Llama-3.2-11B-Vision-Instruct",
"object": "chat.completion", "object": "chat.completion",
"system_fingerprint": "3.1.1-dev0-native", "system_fingerprint": "3.1.1-dev0-native",
"usage": { "usage": {
"completion_tokens": 10, "completion_tokens": 10,
"prompt_tokens": 50, "prompt_tokens": 45,
"total_tokens": 60 "total_tokens": 55
} }
}, },
{ {
@ -32,7 +32,7 @@
"index": 0, "index": 0,
"logprobs": null, "logprobs": null,
"message": { "message": {
"content": "In a small town, a chicken named Cluck", "content": "A chicken sits on a pile of money, looking",
"name": null, "name": null,
"role": "assistant", "role": "assistant",
"tool_calls": null "tool_calls": null
@ -40,15 +40,15 @@
"usage": null "usage": null
} }
], ],
"created": 1738753835, "created": 1739290197,
"id": "", "id": "",
"model": "meta-llama/Llama-3.2-11B-Vision-Instruct", "model": "meta-llama/Llama-3.2-11B-Vision-Instruct",
"object": "chat.completion", "object": "chat.completion",
"system_fingerprint": "3.1.1-dev0-native", "system_fingerprint": "3.1.1-dev0-native",
"usage": { "usage": {
"completion_tokens": 10, "completion_tokens": 10,
"prompt_tokens": 50, "prompt_tokens": 45,
"total_tokens": 60 "total_tokens": 55
} }
} }
] ]

8
integration-tests/models/__snapshots__/test_mllama/test_mllama_simpl.json
View File

@ -5,7 +5,7 @@
"index": 0, "index": 0,
"logprobs": null, "logprobs": null,
"message": { "message": {
"content": "In a small town, a chicken named Cluck", "content": "A chicken sits on a pile of money, looking",
"name": null, "name": null,
"role": "assistant", "role": "assistant",
"tool_calls": null "tool_calls": null
@ -13,14 +13,14 @@
"usage": null "usage": null
} }
], ],
"created": 1738753833, "created": 1739290152,
"id": "", "id": "",
"model": "meta-llama/Llama-3.2-11B-Vision-Instruct", "model": "meta-llama/Llama-3.2-11B-Vision-Instruct",
"object": "chat.completion", "object": "chat.completion",
"system_fingerprint": "3.1.1-dev0-native", "system_fingerprint": "3.1.1-dev0-native",
"usage": { "usage": {
"completion_tokens": 10, "completion_tokens": 10,
"prompt_tokens": 50, "prompt_tokens": 45,
"total_tokens": 60 "total_tokens": 55
} }
} }

13
integration-tests/models/test_mllama.py
View File

@ -28,7 +28,7 @@ async def test_mllama_simpl(mllama, response_snapshot):
"content": [ "content": [
{ {
"type": "text", "type": "text",
"text": "Can you tell me a very short story based on the image?", "text": "Describe the image in 10 words.",
}, },
{ {
"type": "image_url", "type": "image_url",
@ -43,11 +43,12 @@ async def test_mllama_simpl(mllama, response_snapshot):
assert response.usage == { assert response.usage == {
"completion_tokens": 10, "completion_tokens": 10,
"prompt_tokens": 50, "prompt_tokens": 45,
"total_tokens": 60, "total_tokens": 55,
} }
assert ( assert (
response.choices[0].message.content == "In a small town, a chicken named Cluck" response.choices[0].message.content
== "A chicken sits on a pile of money, looking"
) )
assert response == response_snapshot assert response == response_snapshot
@ -65,7 +66,7 @@ async def test_mllama_load(mllama, generate_load, response_snapshot):
"content": [ "content": [
{ {
"type": "text", "type": "text",
"text": "Can you tell me a very short story based on the image?", "text": "Describe the image in 10 words.",
}, },
{ {
"type": "image_url", "type": "image_url",
@ -86,7 +87,7 @@ async def test_mllama_load(mllama, generate_load, response_snapshot):
generated_texts = [response.choices[0].message.content for response in responses] generated_texts = [response.choices[0].message.content for response in responses]
# XXX: TODO: Fix this test. # XXX: TODO: Fix this test.
assert generated_texts[0] == "In a small town, a chicken named Cluck" assert generated_texts[0] == "A chicken sits on a pile of money, looking"
assert len(generated_texts) == 2 assert len(generated_texts) == 2
assert generated_texts, all( assert generated_texts, all(
[text == generated_texts[0] for text in generated_texts] [text == generated_texts[0] for text in generated_texts]