huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-26 20:42:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Updating the old tests.

Browse Source
This commit is contained in:
Nicolas Patry 2025-03-05 18:29:29 +01:00
parent e2c2fc0a49
commit dc024a9432
No known key found for this signature in database
GPG Key ID: 4242CEF24CB6DBF9
14 changed files with 107 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
integration-tests/conftest.py
View File

@ -269,17 +269,17 @@ class ResponseComparator(JSONSnapshotExtension):
def eq_chat_complete_chunk( def eq_chat_complete_chunk(
response: ChatCompletionChunk, other: ChatCompletionChunk response: ChatCompletionChunk, other: ChatCompletionChunk
) -> bool: ) -> bool:
if response.choices[0].delta.content: if response.choices[0].delta.content is not None:
return ( return (
response.choices[0].delta.content == other.choices[0].delta.content response.choices[0].delta.content == other.choices[0].delta.content
) )
elif response.choices[0].delta.tool_calls: elif response.choices[0].delta.tool_calls is not None:
return ( return (
response.choices[0].delta.tool_calls response.choices[0].delta.tool_calls
== other.choices[0].delta.tool_calls == other.choices[0].delta.tool_calls
) )
else: else:
raise RuntimeError("Invalid empty chat chunk") raise RuntimeError(f"Invalid empty chat chunk {response} vs {other}")
def eq_response(response: Response, other: Response) -> bool: def eq_response(response: Response, other: Response) -> bool:
return response.generated_text == other.generated_text and eq_details( return response.generated_text == other.generated_text and eq_details(

4
integration-tests/models/__snapshots__/test_tools_llama/test_flash_llama_grammar_tools.json
View File

@ -26,11 +26,11 @@
"usage": null "usage": null
} }
], ],
"created": 1732293383, "created": 1741195536,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion", "object": "chat.completion",
"system_fingerprint": "2.4.1-dev0-native", "system_fingerprint": "3.1.2-dev0-native",
"usage": { "usage": {
"completion_tokens": 30, "completion_tokens": 30,
"prompt_tokens": 615, "prompt_tokens": 615,

4
integration-tests/models/__snapshots__/test_tools_llama/test_flash_llama_grammar_tools_auto.json
View File

@ -26,11 +26,11 @@
"usage": null "usage": null
} }
], ],
"created": 1732293384, "created": 1741195538,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion", "object": "chat.completion",
"system_fingerprint": "2.4.1-dev0-native", "system_fingerprint": "3.1.2-dev0-native",
"usage": { "usage": {
"completion_tokens": 30, "completion_tokens": 30,
"prompt_tokens": 615, "prompt_tokens": 615,

20
integration-tests/models/__snapshots__/test_tools_llama/test_flash_llama_grammar_tools_choice.json
View File

@ -1,7 +1,7 @@
{ {
"choices": [ "choices": [
{ {
"finish_reason": "eos_token", "finish_reason": "stop",
"index": 0, "index": 0,
"logprobs": null, "logprobs": null,
"message": { "message": {
@ -13,12 +13,12 @@
"function": { "function": {
"arguments": { "arguments": {
"format": "celsius", "format": "celsius",
"location": "New York, NY" "location": "Brooklyn, New York"
}, },
"description": null, "description": null,
"name": "get_current_weather" "name": "get_current_weather"
}, },
"id": 0, "id": "0",
"type": "function" "type": "function"
} }
] ]
@ -26,14 +26,14 @@
"usage": null "usage": null
} }
], ],
"created": 1712852394, "created": 1741195540,
"id": "", "id": "",
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "text_completion", "object": "chat.completion",
"system_fingerprint": "2.0.1-native", "system_fingerprint": "3.1.2-dev0-native",
"usage": { "usage": {
"completion_tokens": 48, "completion_tokens": 30,
"prompt_tokens": 320, "prompt_tokens": 326,
"total_tokens": 368 "total_tokens": 356
} }
} }

10
integration-tests/models/__snapshots__/test_tools_llama/test_flash_llama_grammar_tools_insufficient_information.json
View File

@ -13,14 +13,14 @@
"usage": null "usage": null
} }
], ],
"created": 1728497062, "created": 1741195542,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion", "object": "chat.completion",
"system_fingerprint": "2.4.2-dev0-native", "system_fingerprint": "3.1.2-dev0-native",
"usage": { "usage": {
"completion_tokens": 23, "completion_tokens": 22,
"prompt_tokens": 604, "prompt_tokens": 608,
"total_tokens": 627 "total_tokens": 630
} }
} }

4
integration-tests/models/__snapshots__/test_tools_llama/test_flash_llama_grammar_tools_insufficient_information_stream.json
View File

@ -11,10 +11,10 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1728497531, "created": 1741195542,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
"system_fingerprint": "2.4.2-dev0-native", "system_fingerprint": "3.1.2-dev0-native",
"usage": null "usage": null
} }

60
integration-tests/models/__snapshots__/test_tools_llama/test_flash_llama_grammar_tools_openai.json
View File

@ -24,7 +24,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741183959, "created": 1741195536,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -57,7 +57,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741183959, "created": 1741195536,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -90,7 +90,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741183959, "created": 1741195536,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -123,7 +123,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741183959, "created": 1741195536,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -156,7 +156,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741183959, "created": 1741195536,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -189,7 +189,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741183959, "created": 1741195536,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -222,7 +222,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741183959, "created": 1741195536,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -255,7 +255,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741183959, "created": 1741195536,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -288,7 +288,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741183959, "created": 1741195536,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -321,7 +321,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741183959, "created": 1741195536,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -354,7 +354,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741183959, "created": 1741195536,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -387,7 +387,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741183959, "created": 1741195536,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -420,7 +420,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741183959, "created": 1741195536,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -453,7 +453,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741183959, "created": 1741195536,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -486,7 +486,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741183960, "created": 1741195536,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -519,7 +519,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741183960, "created": 1741195536,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -552,7 +552,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741183960, "created": 1741195536,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -585,7 +585,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741183960, "created": 1741195536,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -618,7 +618,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741183960, "created": 1741195536,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -651,7 +651,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741183960, "created": 1741195536,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -684,7 +684,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741183960, "created": 1741195536,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -717,7 +717,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741183960, "created": 1741195536,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -750,7 +750,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741183960, "created": 1741195536,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -783,7 +783,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741183960, "created": 1741195536,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -816,7 +816,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741183960, "created": 1741195537,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -849,7 +849,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741183960, "created": 1741195537,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -882,7 +882,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741183960, "created": 1741195537,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -915,7 +915,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741183960, "created": 1741195537,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -948,7 +948,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741183960, "created": 1741195537,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -981,7 +981,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741183960, "created": 1741195537,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",

4
integration-tests/models/__snapshots__/test_tools_llama/test_flash_llama_grammar_tools_sea_creatures_stream.json
View File

@ -11,10 +11,10 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1728497461, "created": 1741195545,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
"system_fingerprint": "2.4.2-dev0-native", "system_fingerprint": "3.1.2-dev0-native",
"usage": null "usage": null
} }

8
integration-tests/models/__snapshots__/test_tools_llama/test_flash_llama_grammar_tools_sea_creatures_stream_function_object.json
View File

@ -3,7 +3,8 @@
{ {
"delta": { "delta": {
"role": "assistant", "role": "assistant",
"tool_calls": { "tool_calls": [
{
"function": { "function": {
"arguments": "<|eot_id|>", "arguments": "<|eot_id|>",
"name": null "name": null
@ -12,16 +13,17 @@
"index": 0, "index": 0,
"type": "function" "type": "function"
} }
]
}, },
"finish_reason": "stop", "finish_reason": "stop",
"index": 0, "index": 0,
"logprobs": null "logprobs": null
} }
], ],
"created": 1732293254, "created": 1741195554,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
"system_fingerprint": "2.4.1-dev0-native", "system_fingerprint": "3.1.2-dev0-native",
"usage": null "usage": null
} }

4
integration-tests/models/__snapshots__/test_tools_llama/test_flash_llama_grammar_tools_sea_creatures_stream_none.json
View File

@ -11,10 +11,10 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1729262528, "created": 1741195551,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
"system_fingerprint": "2.3.2-dev0-native", "system_fingerprint": "3.1.2-dev0-native",
"usage": null "usage": null
} }

8
integration-tests/models/__snapshots__/test_tools_llama/test_flash_llama_grammar_tools_sea_creatures_stream_required.json
View File

@ -4,7 +4,8 @@
"delta": { "delta": {
"content": null, "content": null,
"role": "assistant", "role": "assistant",
"tool_calls": { "tool_calls": [
{
"function": { "function": {
"arguments": "<|eot_id|>", "arguments": "<|eot_id|>",
"name": null "name": null
@ -13,16 +14,17 @@
"index": 0, "index": 0,
"type": "function" "type": "function"
} }
]
}, },
"finish_reason": "stop", "finish_reason": "stop",
"index": 0, "index": 0,
"logprobs": null "logprobs": null
} }
], ],
"created": 1732293246, "created": 1741195548,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
"system_fingerprint": "2.4.1-dev0-native", "system_fingerprint": "3.1.2-dev0-native",
"usage": null "usage": null
} }

8
integration-tests/models/__snapshots__/test_tools_llama/test_flash_llama_grammar_tools_stream.json
View File

@ -4,7 +4,8 @@
"delta": { "delta": {
"content": null, "content": null,
"role": "assistant", "role": "assistant",
"tool_calls": { "tool_calls": [
{
"function": { "function": {
"arguments": "<|eot_id|>", "arguments": "<|eot_id|>",
"name": null "name": null
@ -13,16 +14,17 @@
"index": 0, "index": 0,
"type": "function" "type": "function"
} }
]
}, },
"finish_reason": "stop", "finish_reason": "stop",
"index": 0, "index": 0,
"logprobs": null "logprobs": null
} }
], ],
"created": 1732293235, "created": 1741195541,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
"system_fingerprint": "2.4.1-dev0-native", "system_fingerprint": "3.1.2-dev0-native",
"usage": null "usage": null
} }

4
integration-tests/models/__snapshots__/test_tools_llama/test_flash_llama_tool_reply_response.json
View File

@ -13,11 +13,11 @@
"usage": null "usage": null
} }
], ],
"created": 1739932427, "created": 1741195556,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion", "object": "chat.completion",
"system_fingerprint": "3.1.1-dev0-native", "system_fingerprint": "3.1.2-dev0-native",
"usage": { "usage": {
"completion_tokens": 79, "completion_tokens": 79,
"prompt_tokens": 103, "prompt_tokens": 103,

12
integration-tests/models/test_tools_llama.py
View File

@ -246,7 +246,9 @@ async def test_flash_llama_grammar_tools_stream(
last_response = None last_response = None
async for response in responses: async for response in responses:
count += 1 count += 1
tool_calls_generated += response.choices[0].delta.tool_calls.function.arguments tool_calls_generated += (
response.choices[0].delta.tool_calls[0].function.arguments
)
last_response = response last_response = response
assert response.choices[0].delta.content is None assert response.choices[0].delta.content is None
@ -393,7 +395,9 @@ async def test_flash_llama_grammar_tools_sea_creatures_stream_required(
async for response in responses: async for response in responses:
count += 1 count += 1
assert response.choices[0].delta.content is None assert response.choices[0].delta.content is None
tool_calls_generated += response.choices[0].delta.tool_calls.function.arguments tool_calls_generated += (
response.choices[0].delta.tool_calls[0].function.arguments
)
last_response = response last_response = response
assert count == 29 assert count == 29
@ -491,8 +495,8 @@ async def test_flash_llama_grammar_tools_sea_creatures_stream_function_object(
break break
response = json.loads(line) response = json.loads(line)
tool_calls_generated += response["choices"][0]["delta"]["tool_calls"][ tool_calls_generated += response["choices"][0]["delta"]["tool_calls"][
"function" 0
]["arguments"] ]["function"]["arguments"]
last_response = response last_response = response
assert count == 39 assert count == 39