huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-19 22:02:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix tool call2 (#3076)

Browse Source 
* Making `tool_calls` a vector.

* Arguments output is a string.

* Update all the integration tests.

* Add the requirements.

* Upgrade other tests.

* Clippy.

* Update the old test.
This commit is contained in:
Nicolas Patry 2025-03-07 19:45:57 +01:00 committed by GitHub
parent 55a6618434
commit 622908deab
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
22 changed files with 6054 additions and 545 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
integration-tests/conftest.py
View File

@ -1,6 +1,13 @@
pytest_plugins = ["fixtures.neuron.service", "fixtures.neuron.export_models"] pytest_plugins = ["fixtures.neuron.service", "fixtures.neuron.export_models"]
# ruff: noqa: E402 # ruff: noqa: E402
from _pytest.fixtures import SubRequest from _pytest.fixtures import SubRequest
from huggingface_hub.inference._generated.types.chat_completion import (
ChatCompletionStreamOutput,
ChatCompletionOutput,
)
from openai.types.chat.chat_completion_chunk import (
ChatCompletionChunk as OAIChatCompletionChunk,
)
import requests import requests
@ -115,6 +122,31 @@ class ResponseComparator(JSONSnapshotExtension):
rtol = 0.2 rtol = 0.2
ignore_logprob = False ignore_logprob = False
def _serialize(
self,
data,
):
if (
isinstance(data, Response)
or isinstance(data, ChatComplete)
or isinstance(data, ChatCompletionChunk)
or isinstance(data, ChatCompletionComplete)
or isinstance(data, Completion)
or isinstance(data, OAIChatCompletionChunk)
):
data = data.model_dump()
elif isinstance(data, ChatCompletionStreamOutput) or isinstance(
data, ChatCompletionOutput
):
data = dict(data)
elif isinstance(data, List):
data = [self._serialize(d) for d in data]
elif isinstance(data, dict):
return data
else:
raise RuntimeError(f"Unexpected data {type(data)} : {data}")
return data
def serialize( def serialize(
self, self,
data, data,
@ -123,17 +155,7 @@ class ResponseComparator(JSONSnapshotExtension):
exclude=None, exclude=None,
matcher=None, matcher=None,
): ):
if ( data = self._serialize(data)
isinstance(data, Response)
or isinstance(data, ChatComplete)
or isinstance(data, ChatCompletionChunk)
or isinstance(data, ChatCompletionComplete)
):
data = data.model_dump()
if isinstance(data, List):
data = [d.model_dump() for d in data]
data = self._filter( data = self._filter(
data=data, data=data,
depth=0, depth=0,
@ -142,7 +164,8 @@ class ResponseComparator(JSONSnapshotExtension):
include=include, include=include,
matcher=matcher, matcher=matcher,
) )
return json.dumps(data, indent=2, ensure_ascii=False, sort_keys=False) + "\n" data = json.dumps(data, indent=2, ensure_ascii=False, sort_keys=False) + "\n"
return data
def matches( def matches(
self, self,
@ -158,7 +181,7 @@ class ResponseComparator(JSONSnapshotExtension):
if isinstance(data, Dict): if isinstance(data, Dict):
if "choices" in data: if "choices" in data:
data["choices"] = list( data["choices"] = list(
sorted(data["choices"], key=lambda x: x["index"]) sorted(data["choices"], key=lambda x: int(x["index"]))
) )
choices = data["choices"] choices = data["choices"]
if isinstance(choices, List) and len(choices) >= 1: if isinstance(choices, List) and len(choices) >= 1:
@ -171,7 +194,7 @@ class ResponseComparator(JSONSnapshotExtension):
return Response(**data) return Response(**data)
if isinstance(data, List): if isinstance(data, List):
return [_convert_data(d) for d in data] return [_convert_data(d) for d in data]
raise NotImplementedError raise NotImplementedError(f"Data: {data}")
def eq_token(token: Token, other: Token) -> bool: def eq_token(token: Token, other: Token) -> bool:
return ( return (
@ -269,17 +292,25 @@ class ResponseComparator(JSONSnapshotExtension):
def eq_chat_complete_chunk( def eq_chat_complete_chunk(
response: ChatCompletionChunk, other: ChatCompletionChunk response: ChatCompletionChunk, other: ChatCompletionChunk
) -> bool: ) -> bool:
if response.choices[0].delta.content is not None: if response.choices:
return ( if response.choices[0].delta.content is not None:
response.choices[0].delta.content == other.choices[0].delta.content return (
) response.choices[0].delta.content
elif response.choices[0].delta.tool_calls is not None: == other.choices[0].delta.content
return ( )
response.choices[0].delta.tool_calls elif response.choices[0].delta.tool_calls is not None:
== other.choices[0].delta.tool_calls return (
) response.choices[0].delta.tool_calls
== other.choices[0].delta.tool_calls
)
else:
raise RuntimeError(
f"Invalid empty chat chunk {response} vs {other}"
)
elif response.usage is not None:
return response.usage == other.usage
else: else:
raise RuntimeError(f"Invalid empty chat chunk {response} vs {other}") raise RuntimeError(f"Invalid empty chat {response} vs {other}")
def eq_response(response: Response, other: Response) -> bool: def eq_response(response: Response, other: Response) -> bool:
return response.generated_text == other.generated_text and eq_details( return response.generated_text == other.generated_text and eq_details(
@ -294,6 +325,9 @@ class ResponseComparator(JSONSnapshotExtension):
if not isinstance(snapshot_data, List): if not isinstance(snapshot_data, List):
snapshot_data = [snapshot_data] snapshot_data = [snapshot_data]
if len(serialized_data) == 0:
return len(snapshot_data) == len(serialized_data)
if isinstance(serialized_data[0], Completion): if isinstance(serialized_data[0], Completion):
return len(snapshot_data) == len(serialized_data) and all( return len(snapshot_data) == len(serialized_data) and all(
[eq_completion(r, o) for r, o in zip(serialized_data, snapshot_data)] [eq_completion(r, o) for r, o in zip(serialized_data, snapshot_data)]

60
integration-tests/models/__snapshots__/test_completion_prompts/test_flash_llama_completion_stream_usage.json
View File

@ -12,11 +12,11 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1726656043, "created": 1741338471,
"id": "", "id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
"system_fingerprint": "2.2.1-dev0-native", "system_fingerprint": "3.1.2-dev0-native",
"usage": null "usage": null
}, },
{ {
@ -32,11 +32,11 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1726656043, "created": 1741338471,
"id": "", "id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
"system_fingerprint": "2.2.1-dev0-native", "system_fingerprint": "3.1.2-dev0-native",
"usage": null "usage": null
}, },
{ {
@ -52,11 +52,11 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1726656043, "created": 1741338471,
"id": "", "id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
"system_fingerprint": "2.2.1-dev0-native", "system_fingerprint": "3.1.2-dev0-native",
"usage": null "usage": null
}, },
{ {
@ -72,11 +72,11 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1726656043, "created": 1741338471,
"id": "", "id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
"system_fingerprint": "2.2.1-dev0-native", "system_fingerprint": "3.1.2-dev0-native",
"usage": null "usage": null
}, },
{ {
@ -92,11 +92,11 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1726656043, "created": 1741338472,
"id": "", "id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
"system_fingerprint": "2.2.1-dev0-native", "system_fingerprint": "3.1.2-dev0-native",
"usage": null "usage": null
}, },
{ {
@ -112,11 +112,11 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1726656043, "created": 1741338472,
"id": "", "id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
"system_fingerprint": "2.2.1-dev0-native", "system_fingerprint": "3.1.2-dev0-native",
"usage": null "usage": null
}, },
{ {
@ -132,11 +132,11 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1726656044, "created": 1741338472,
"id": "", "id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
"system_fingerprint": "2.2.1-dev0-native", "system_fingerprint": "3.1.2-dev0-native",
"usage": null "usage": null
}, },
{ {
@ -152,11 +152,11 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1726656044, "created": 1741338472,
"id": "", "id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
"system_fingerprint": "2.2.1-dev0-native", "system_fingerprint": "3.1.2-dev0-native",
"usage": null "usage": null
}, },
{ {
@ -172,11 +172,11 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1726656044, "created": 1741338472,
"id": "", "id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
"system_fingerprint": "2.2.1-dev0-native", "system_fingerprint": "3.1.2-dev0-native",
"usage": null "usage": null
}, },
{ {
@ -192,11 +192,11 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1726656044, "created": 1741338472,
"id": "", "id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
"system_fingerprint": "2.2.1-dev0-native", "system_fingerprint": "3.1.2-dev0-native",
"usage": { "usage": {
"completion_tokens": 10, "completion_tokens": 10,
"prompt_tokens": 40, "prompt_tokens": 40,

17
integration-tests/models/__snapshots__/test_tools_llama/test_flash_llama_grammar_tools.json → integration-tests/models/__snapshots__/test_tools_llama/test_flash_llama_grammar_tools_auto_nostream.json
View File

@ -6,15 +6,11 @@
"logprobs": null, "logprobs": null,
"message": { "message": {
"content": null, "content": null,
"name": null,
"role": "assistant", "role": "assistant",
"tool_calls": [ "tool_calls": [
{ {
"function": { "function": {
"arguments": { "arguments": "{\"format\":\"fahrenheit\",\"location\":\"Brooklyn, NY\"}",
"format": "celsius",
"location": "Brooklyn, New York"
},
"description": null, "description": null,
"name": "get_current_weather" "name": "get_current_weather"
}, },
@ -22,18 +18,17 @@
"type": "function" "type": "function"
} }
] ]
}, }
"usage": null
} }
], ],
"created": 1741195536, "created": 1741263682,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion", "object": "chat.completion",
"system_fingerprint": "3.1.2-dev0-native", "system_fingerprint": "3.1.2-dev0-native",
"usage": { "usage": {
"completion_tokens": 30, "completion_tokens": 29,
"prompt_tokens": 615, "prompt_tokens": 501,
"total_tokens": 645 "total_tokens": 530
} }
} }

17
integration-tests/models/__snapshots__/test_tools_llama/test_flash_llama_grammar_tools_auto.json → integration-tests/models/__snapshots__/test_tools_llama/test_flash_llama_grammar_tools_choice_nostream.json
View File

@ -6,15 +6,11 @@
"logprobs": null, "logprobs": null,
"message": { "message": {
"content": null, "content": null,
"name": null,
"role": "assistant", "role": "assistant",
"tool_calls": [ "tool_calls": [
{ {
"function": { "function": {
"arguments": { "arguments": "{\"format\":\"fahrenheit\",\"location\":\"Brooklyn, NY\"}",
"format": "celsius",
"location": "Brooklyn, New York"
},
"description": null, "description": null,
"name": "get_current_weather" "name": "get_current_weather"
}, },
@ -22,18 +18,17 @@
"type": "function" "type": "function"
} }
] ]
}, }
"usage": null
} }
], ],
"created": 1741195538, "created": 1741263684,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion", "object": "chat.completion",
"system_fingerprint": "3.1.2-dev0-native", "system_fingerprint": "3.1.2-dev0-native",
"usage": { "usage": {
"completion_tokens": 30, "completion_tokens": 29,
"prompt_tokens": 615, "prompt_tokens": 286,
"total_tokens": 645 "total_tokens": 315
} }
} }

842
integration-tests/models/__snapshots__/test_tools_llama/test_flash_llama_grammar_tools_choice_stream.json Normal file
View File

@ -0,0 +1,842 @@
[
{
"choices": [
{
"delta": {
"content": null,
"role": "assistant",
"tool_calls": [
{
"function": {
"arguments": "{\"",
"name": null
},
"id": "",
"index": 0,
"type": "function"
}
]
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 1741263685,
"id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "3.1.2-dev0-native",
"usage": null
},
{
"choices": [
{
"delta": {
"content": null,
"role": "assistant",
"tool_calls": [
{
"function": {
"arguments": "function",
"name": null
},
"id": "",
"index": 0,
"type": "function"
}
]
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 1741263685,
"id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "3.1.2-dev0-native",
"usage": null
},
{
"choices": [
{
"delta": {
"content": null,
"role": "assistant",
"tool_calls": [
{
"function": {
"arguments": "\":",
"name": null
},
"id": "",
"index": 0,
"type": "function"
}
]
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 1741263685,
"id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "3.1.2-dev0-native",
"usage": null
},
{
"choices": [
{
"delta": {
"content": null,
"role": "assistant",
"tool_calls": [
{
"function": {
"arguments": " {\"",
"name": null
},
"id": "",
"index": 0,
"type": "function"
}
]
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 1741263685,
"id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "3.1.2-dev0-native",
"usage": null
},
{
"choices": [
{
"delta": {
"content": null,
"role": "assistant",
"tool_calls": [
{
"function": {
"arguments": "_",
"name": null
},
"id": "",
"index": 0,
"type": "function"
}
]
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 1741263685,
"id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "3.1.2-dev0-native",
"usage": null
},
{
"choices": [
{
"delta": {
"content": null,
"role": "assistant",
"tool_calls": [
{
"function": {
"arguments": "name",
"name": null
},
"id": "",
"index": 0,
"type": "function"
}
]
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 1741263685,
"id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "3.1.2-dev0-native",
"usage": null
},
{
"choices": [
{
"delta": {
"content": null,
"role": "assistant",
"tool_calls": [
{
"function": {
"arguments": "\":",
"name": null
},
"id": "",
"index": 0,
"type": "function"
}
]
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 1741263685,
"id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "3.1.2-dev0-native",
"usage": null
},
{
"choices": [
{
"delta": {
"content": null,
"role": "assistant",
"tool_calls": [
{
"function": {
"arguments": " \"",
"name": null
},
"id": "",
"index": 0,
"type": "function"
}
]
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 1741263685,
"id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "3.1.2-dev0-native",
"usage": null
},
{
"choices": [
{
"delta": {
"content": null,
"role": "assistant",
"tool_calls": [
{
"function": {
"arguments": "get",
"name": null
},
"id": "",
"index": 0,
"type": "function"
}
]
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 1741263685,
"id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "3.1.2-dev0-native",
"usage": null
},
{
"choices": [
{
"delta": {
"content": null,
"role": "assistant",
"tool_calls": [
{
"function": {
"arguments": "_current",
"name": null
},
"id": "",
"index": 0,
"type": "function"
}
]
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 1741263685,
"id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "3.1.2-dev0-native",
"usage": null
},
{
"choices": [
{
"delta": {
"content": null,
"role": "assistant",
"tool_calls": [
{
"function": {
"arguments": "_weather",
"name": null
},
"id": "",
"index": 0,
"type": "function"
}
]
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 1741263685,
"id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "3.1.2-dev0-native",
"usage": null
},
{
"choices": [
{
"delta": {
"content": null,
"role": "assistant",
"tool_calls": [
{
"function": {
"arguments": "\",",
"name": null
},
"id": "",
"index": 0,
"type": "function"
}
]
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 1741263685,
"id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "3.1.2-dev0-native",
"usage": null
},
{
"choices": [
{
"delta": {
"content": null,
"role": "assistant",
"tool_calls": [
{
"function": {
"arguments": " \"",
"name": null
},
"id": "",
"index": 0,
"type": "function"
}
]
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 1741263685,
"id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "3.1.2-dev0-native",
"usage": null
},
{
"choices": [
{
"delta": {
"content": null,
"role": "assistant",
"tool_calls": [
{
"function": {
"arguments": "location",
"name": null
},
"id": "",
"index": 0,
"type": "function"
}
]
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 1741263685,
"id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "3.1.2-dev0-native",
"usage": null
},
{
"choices": [
{
"delta": {
"content": null,
"role": "assistant",
"tool_calls": [
{
"function": {
"arguments": "\":",
"name": null
},
"id": "",
"index": 0,
"type": "function"
}
]
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 1741263685,
"id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "3.1.2-dev0-native",
"usage": null
},
{
"choices": [
{
"delta": {
"content": null,
"role": "assistant",
"tool_calls": [
{
"function": {
"arguments": " \"",
"name": null
},
"id": "",
"index": 0,
"type": "function"
}
]
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 1741263685,
"id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "3.1.2-dev0-native",
"usage": null
},
{
"choices": [
{
"delta": {
"content": null,
"role": "assistant",
"tool_calls": [
{
"function": {
"arguments": "Paris",
"name": null
},
"id": "",
"index": 0,
"type": "function"
}
]
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 1741263685,
"id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "3.1.2-dev0-native",
"usage": null
},
{
"choices": [
{
"delta": {
"content": null,
"role": "assistant",
"tool_calls": [
{
"function": {
"arguments": ",",
"name": null
},
"id": "",
"index": 0,
"type": "function"
}
]
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 1741263685,
"id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "3.1.2-dev0-native",
"usage": null
},
{
"choices": [
{
"delta": {
"content": null,
"role": "assistant",
"tool_calls": [
{
"function": {
"arguments": " France",
"name": null
},
"id": "",
"index": 0,
"type": "function"
}
]
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 1741263685,
"id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "3.1.2-dev0-native",
"usage": null
},
{
"choices": [
{
"delta": {
"content": null,
"role": "assistant",
"tool_calls": [
{
"function": {
"arguments": "\",",
"name": null
},
"id": "",
"index": 0,
"type": "function"
}
]
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 1741263685,
"id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "3.1.2-dev0-native",
"usage": null
},
{
"choices": [
{
"delta": {
"content": null,
"role": "assistant",
"tool_calls": [
{
"function": {
"arguments": " \"",
"name": null
},
"id": "",
"index": 0,
"type": "function"
}
]
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 1741263685,
"id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "3.1.2-dev0-native",
"usage": null
},
{
"choices": [
{
"delta": {
"content": null,
"role": "assistant",
"tool_calls": [
{
"function": {
"arguments": "format",
"name": null
},
"id": "",
"index": 0,
"type": "function"
}
]
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 1741263685,
"id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "3.1.2-dev0-native",
"usage": null
},
{
"choices": [
{
"delta": {
"content": null,
"role": "assistant",
"tool_calls": [
{
"function": {
"arguments": "\":",
"name": null
},
"id": "",
"index": 0,
"type": "function"
}
]
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 1741263685,
"id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "3.1.2-dev0-native",
"usage": null
},
{
"choices": [
{
"delta": {
"content": null,
"role": "assistant",
"tool_calls": [
{
"function": {
"arguments": " \"",
"name": null
},
"id": "",
"index": 0,
"type": "function"
}
]
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 1741263685,
"id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "3.1.2-dev0-native",
"usage": null
},
{
"choices": [
{
"delta": {
"content": null,
"role": "assistant",
"tool_calls": [
{
"function": {
"arguments": "c",
"name": null
},
"id": "",
"index": 0,
"type": "function"
}
]
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 1741263685,
"id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "3.1.2-dev0-native",
"usage": null
},
{
"choices": [
{
"delta": {
"content": null,
"role": "assistant",
"tool_calls": [
{
"function": {
"arguments": "elsius",
"name": null
},
"id": "",
"index": 0,
"type": "function"
}
]
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 1741263685,
"id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "3.1.2-dev0-native",
"usage": null
},
{
"choices": [
{
"delta": {
"content": null,
"role": "assistant",
"tool_calls": [
{
"function": {
"arguments": "\"}}",
"name": null
},
"id": "",
"index": 0,
"type": "function"
}
]
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 1741263685,
"id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "3.1.2-dev0-native",
"usage": null
},
{
"choices": [
{
"delta": {
"content": null,
"role": "assistant",
"tool_calls": [
{
"function": {
"arguments": "<|eot_id|>",
"name": null
},
"id": "",
"index": 0,
"type": "function"
}
]
},
"finish_reason": "stop",
"index": 0,
"logprobs": null
}
],
"created": 1741263685,
"id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "3.1.2-dev0-native",
"usage": null
}
]

14
integration-tests/models/__snapshots__/test_tools_llama/test_flash_llama_grammar_tools_insufficient_information.json → integration-tests/models/__snapshots__/test_tools_llama/test_flash_llama_grammar_tools_insufficient_information_nostream.json
View File

@ -5,22 +5,20 @@
"index": 0, "index": 0,
"logprobs": null, "logprobs": null,
"message": { "message": {
"content": "I am an AI assistant", "content": "I am a helpful assistant!",
"name": null,
"role": "assistant", "role": "assistant",
"tool_calls": null "tool_calls": null
}, }
"usage": null
} }
], ],
"created": 1741195542, "created": 1741263686,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion", "object": "chat.completion",
"system_fingerprint": "3.1.2-dev0-native", "system_fingerprint": "3.1.2-dev0-native",
"usage": { "usage": {
"completion_tokens": 22, "completion_tokens": 23,
"prompt_tokens": 608, "prompt_tokens": 494,
"total_tokens": 630 "total_tokens": 517
} }
} }

122
integration-tests/models/__snapshots__/test_tools_llama/test_flash_llama_grammar_tools_insufficient_information_stream.json
View File

@ -1,20 +1,102 @@
{ [
"choices": [ {
{ "choices": [
"delta": { {
"content": " assistant", "delta": {
"role": "assistant", "content": "I",
"tool_calls": null "role": "assistant",
}, "tool_calls": null
"finish_reason": null, },
"index": 0, "finish_reason": null,
"logprobs": null "index": 0,
} "logprobs": null
], }
"created": 1741195542, ],
"id": "", "created": 1741263687,
"model": "meta-llama/Llama-3.1-8B-Instruct", "id": "",
"object": "chat.completion.chunk", "model": "meta-llama/Llama-3.1-8B-Instruct",
"system_fingerprint": "3.1.2-dev0-native", "object": "chat.completion.chunk",
"usage": null "system_fingerprint": "3.1.2-dev0-native",
} "usage": null
},
{
"choices": [
{
"delta": {
"content": " am",
"role": "assistant",
"tool_calls": null
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 1741263687,
"id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "3.1.2-dev0-native",
"usage": null
},
{
"choices": [
{
"delta": {
"content": " a",
"role": "assistant",
"tool_calls": null
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 1741263687,
"id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "3.1.2-dev0-native",
"usage": null
},
{
"choices": [
{
"delta": {
"content": " helpful",
"role": "assistant",
"tool_calls": null
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 1741263687,
"id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "3.1.2-dev0-native",
"usage": null
},
{
"choices": [
{
"delta": {
"content": " assistant",
"role": "assistant",
"tool_calls": null
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 1741263687,
"id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "3.1.2-dev0-native",
"usage": null
}
]

17
integration-tests/models/__snapshots__/test_tools_llama/test_flash_llama_grammar_tools_choice.json → integration-tests/models/__snapshots__/test_tools_llama/test_flash_llama_grammar_tools_nostream.json
View File

@ -6,15 +6,11 @@
"logprobs": null, "logprobs": null,
"message": { "message": {
"content": null, "content": null,
"name": null,
"role": "assistant", "role": "assistant",
"tool_calls": [ "tool_calls": [
{ {
"function": { "function": {
"arguments": { "arguments": "{\"format\":\"fahrenheit\",\"location\":\"Brooklyn, NY\"}",
"format": "celsius",
"location": "Brooklyn, New York"
},
"description": null, "description": null,
"name": "get_current_weather" "name": "get_current_weather"
}, },
@ -22,18 +18,17 @@
"type": "function" "type": "function"
} }
] ]
}, }
"usage": null
} }
], ],
"created": 1741195540, "created": 1741263680,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion", "object": "chat.completion",
"system_fingerprint": "3.1.2-dev0-native", "system_fingerprint": "3.1.2-dev0-native",
"usage": { "usage": {
"completion_tokens": 30, "completion_tokens": 29,
"prompt_tokens": 326, "prompt_tokens": 501,
"total_tokens": 356 "total_tokens": 530
} }
} }

97
integration-tests/models/__snapshots__/test_tools_llama/test_flash_llama_grammar_tools_openai.json
View File

@ -24,7 +24,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741195536, "created": 1741263681,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -57,7 +57,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741195536, "created": 1741263681,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -90,7 +90,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741195536, "created": 1741263681,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -123,7 +123,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741195536, "created": 1741263681,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -156,7 +156,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741195536, "created": 1741263681,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -189,7 +189,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741195536, "created": 1741263681,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -222,7 +222,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741195536, "created": 1741263681,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -255,7 +255,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741195536, "created": 1741263681,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -288,7 +288,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741195536, "created": 1741263681,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -321,7 +321,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741195536, "created": 1741263681,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -354,7 +354,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741195536, "created": 1741263681,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -387,7 +387,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741195536, "created": 1741263681,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -420,7 +420,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741195536, "created": 1741263681,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -453,7 +453,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741195536, "created": 1741263681,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -486,7 +486,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741195536, "created": 1741263681,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -519,7 +519,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741195536, "created": 1741263681,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -552,7 +552,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741195536, "created": 1741263681,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -585,7 +585,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741195536, "created": 1741263681,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -618,7 +618,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741195536, "created": 1741263681,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -637,7 +637,7 @@
"tool_calls": [ "tool_calls": [
{ {
"function": { "function": {
"arguments": " New", "arguments": " NY",
"name": null "name": null
}, },
"id": "", "id": "",
@ -651,40 +651,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741195536, "created": 1741263681,
"id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"service_tier": null,
"system_fingerprint": "3.1.2-dev0-native",
"usage": null
},
{
"choices": [
{
"delta": {
"content": null,
"function_call": null,
"refusal": null,
"role": "assistant",
"tool_calls": [
{
"function": {
"arguments": " York",
"name": null
},
"id": "",
"index": 0,
"type": "function"
}
]
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 1741195536,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -717,7 +684,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741195536, "created": 1741263681,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -750,7 +717,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741195536, "created": 1741263681,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -783,7 +750,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741195536, "created": 1741263681,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -816,7 +783,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741195537, "created": 1741263681,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -849,7 +816,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741195537, "created": 1741263681,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -868,7 +835,7 @@
"tool_calls": [ "tool_calls": [
{ {
"function": { "function": {
"arguments": "c", "arguments": "f",
"name": null "name": null
}, },
"id": "", "id": "",
@ -882,7 +849,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741195537, "created": 1741263681,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -901,7 +868,7 @@
"tool_calls": [ "tool_calls": [
{ {
"function": { "function": {
"arguments": "elsius", "arguments": "ahrenheit",
"name": null "name": null
}, },
"id": "", "id": "",
@ -915,7 +882,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741195537, "created": 1741263681,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -948,7 +915,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741195537, "created": 1741263681,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -981,7 +948,7 @@
"logprobs": null "logprobs": null
} }
], ],
"created": 1741195537, "created": 1741263681,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",

20
integration-tests/models/__snapshots__/test_tools_llama/test_flash_llama_grammar_tools_sea_creatures_stream.json
View File

@ -1,20 +0,0 @@
{
"choices": [
{
"delta": {
"content": " fans",
"role": "assistant",
"tool_calls": null
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 1741195545,
"id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "3.1.2-dev0-native",
"usage": null
}

1542
integration-tests/models/__snapshots__/test_tools_llama/test_flash_llama_grammar_tools_sea_creatures_stream_auto.json Normal file
View File

File diff suppressed because it is too large Load Diff

1261
integration-tests/models/__snapshots__/test_tools_llama/test_flash_llama_grammar_tools_sea_creatures_stream_function_object.json
View File

File diff suppressed because it is too large Load Diff

2022
integration-tests/models/__snapshots__/test_tools_llama/test_flash_llama_grammar_tools_sea_creatures_stream_none.json
View File

File diff suppressed because it is too large Load Diff

31
integration-tests/models/__snapshots__/test_tools_llama/test_flash_llama_grammar_tools_sea_creatures_stream_required.json
View File

@ -1,30 +1 @@
{ []
"choices": [
{
"delta": {
"content": null,
"role": "assistant",
"tool_calls": [
{
"function": {
"arguments": "<|eot_id|>",
"name": null
},
"id": "",
"index": 0,
"type": "function"
}
]
},
"finish_reason": "stop",
"index": 0,
"logprobs": null
}
],
"created": 1741195548,
"id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "3.1.2-dev0-native",
"usage": null
}

30
integration-tests/models/__snapshots__/test_tools_llama/test_flash_llama_grammar_tools_stream.json
View File

@ -1,30 +0,0 @@
{
"choices": [
{
"delta": {
"content": null,
"role": "assistant",
"tool_calls": [
{
"function": {
"arguments": "<|eot_id|>",
"name": null
},
"id": "",
"index": 0,
"type": "function"
}
]
},
"finish_reason": "stop",
"index": 0,
"logprobs": null
}
],
"created": 1741195541,
"id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "3.1.2-dev0-native",
"usage": null
}

14
integration-tests/models/__snapshots__/test_tools_llama/test_flash_llama_tool_reply_response.json
View File

@ -5,22 +5,20 @@
"index": 0, "index": 0,
"logprobs": null, "logprobs": null,
"message": { "message": {
"content": "I can't access real-time data, but I can provide you with current conditions and forecast for Paris, France:\n\nThe current conditions in Paris are mostly cloudy with a temperature of 6.7°C (44.1°F). \n\nPlease note that the actual weather may differ from this information, and I recommend checking the forecast on a reliable weather website for the most up-to-date information.", "content": "I can't access real-time data, but I can provide you with current conditions and forecast for Paris, France:\n\nThe current conditions in Paris are mostly cloudy with a temperature of 6.7°C (44.1°F). \n\nPlease note that the actual weather may differ from the provided information. For up-to-date information, I suggest checking a reliable weather website or app for the latest conditions and forecast.",
"name": null,
"role": "assistant", "role": "assistant",
"tool_calls": null "tool_calls": null
}, }
"usage": null
} }
], ],
"created": 1741195556, "created": 1741263702,
"id": "", "id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct", "model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion", "object": "chat.completion",
"system_fingerprint": "3.1.2-dev0-native", "system_fingerprint": "3.1.2-dev0-native",
"usage": { "usage": {
"completion_tokens": 79, "completion_tokens": 83,
"prompt_tokens": 103, "prompt_tokens": 109,
"total_tokens": 182 "total_tokens": 192
} }
} }

112
integration-tests/models/test_completion_prompts.py
View File

@ -2,8 +2,9 @@ import pytest
import requests import requests
import json import json
from aiohttp import ClientSession from aiohttp import ClientSession
from huggingface_hub import InferenceClient
from text_generation.types import Completion, ChatCompletionChunk from text_generation.types import Completion
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
@ -52,52 +53,35 @@ def test_flash_llama_completion_single_prompt(
async def test_flash_llama_completion_stream_usage( async def test_flash_llama_completion_stream_usage(
flash_llama_completion, response_snapshot flash_llama_completion, response_snapshot
): ):
url = f"{flash_llama_completion.base_url}/v1/chat/completions" client = InferenceClient(base_url=f"{flash_llama_completion.base_url}/v1")
request = { stream = client.chat_completion(
"model": "tgi", model="tgi",
"messages": [ messages=[
{ {
"role": "user", "role": "user",
"content": "What is Deep Learning?", "content": "What is Deep Learning?",
} }
], ],
"max_tokens": 10, max_tokens=10,
"temperature": 0.0, temperature=0.0,
"stream_options": {"include_usage": True}, stream_options={"include_usage": True},
"stream": True, stream=True,
} )
string = "" string = ""
chunks = [] chunks = []
had_usage = False had_usage = False
async with ClientSession(headers=flash_llama_completion.headers) as session: for chunk in stream:
async with session.post(url, json=request) as response: # remove "data:"
# iterate over the stream chunks.append(chunk)
async for chunk in response.content.iter_any(): print(f"Chunk {chunk}")
# remove "data:" if len(chunk.choices) == 1:
chunk = chunk.decode().split("\n\n") index = chunk.choices[0].index
# remove "data:" if present assert index == 0
chunk = [c.replace("data:", "") for c in chunk] string += chunk.choices[0].delta.content
# remove empty strings if chunk.usage:
chunk = [c for c in chunk if c] assert not had_usage
# remove completion marking chunk had_usage = True
chunk = [c for c in chunk if c != " [DONE]"]
# parse json
chunk = [json.loads(c) for c in chunk]
for c in chunk:
chunks.append(ChatCompletionChunk(**c))
assert "choices" in c
if len(c["choices"]) == 1:
index = c["choices"][0]["index"]
assert index == 0
string += c["choices"][0]["delta"]["content"]
has_usage = c["usage"] is not None
assert not had_usage
if has_usage:
had_usage = True
else:
raise RuntimeError("Expected different payload")
assert had_usage assert had_usage
assert ( assert (
string string
@ -105,51 +89,29 @@ async def test_flash_llama_completion_stream_usage(
) )
assert chunks == response_snapshot assert chunks == response_snapshot
request = { stream = client.chat_completion(
"model": "tgi", model="tgi",
"messages": [ messages=[
{ {
"role": "user", "role": "user",
"content": "What is Deep Learning?", "content": "What is Deep Learning?",
} }
], ],
"max_tokens": 10, max_tokens=10,
"temperature": 0.0, temperature=0.0,
"stream": True, # No usage
} # stream_options={"include_usage": True},
stream=True,
)
string = "" string = ""
chunks = [] chunks = []
had_usage = False had_usage = False
async with ClientSession(headers=flash_llama_completion.headers) as session: for chunk in stream:
async with session.post(url, json=request) as response: chunks.append(chunk)
# iterate over the stream assert chunk.usage is None
async for chunk in response.content.iter_any(): assert len(chunk.choices) == 1
# remove "data:" assert chunk.choices[0].index == 0
chunk = chunk.decode().split("\n\n") string += chunk.choices[0].delta.content
# remove "data:" if present
chunk = [c.replace("data:", "") for c in chunk]
# remove empty strings
chunk = [c for c in chunk if c]
# remove completion marking chunk
chunk = [c for c in chunk if c != " [DONE]"]
# parse json
chunk = [json.loads(c) for c in chunk]
for c in chunk:
chunks.append(ChatCompletionChunk(**c))
assert "choices" in c
if len(c["choices"]) == 1:
index = c["choices"][0]["index"]
assert index == 0
string += c["choices"][0]["delta"]["content"]
has_usage = c["usage"] is not None
assert not had_usage
if has_usage:
had_usage = True
else:
raise RuntimeError("Expected different payload")
assert not had_usage
assert ( assert (
string string
== "**Deep Learning: An Overview**\n=====================================\n\n" == "**Deep Learning: An Overview**\n=====================================\n\n"

279
integration-tests/models/test_tools_llama.py
View File

@ -1,7 +1,10 @@
import pytest import pytest
import requests
import json
from openai import OpenAI from openai import OpenAI
from huggingface_hub import InferenceClient
from huggingface_hub.inference._generated.types.chat_completion import (
ChatCompletionOutputToolCall,
ChatCompletionOutputFunctionDefinition,
)
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
@ -77,8 +80,11 @@ tools = [
@pytest.mark.asyncio @pytest.mark.asyncio
@pytest.mark.private @pytest.mark.private
async def test_flash_llama_grammar_tools(flash_llama_grammar_tools, response_snapshot): async def test_flash_llama_grammar_tools_nostream(
response = await flash_llama_grammar_tools.chat( flash_llama_grammar_tools, response_snapshot
):
client = InferenceClient(base_url=f"{flash_llama_grammar_tools.base_url}/v1")
response = client.chat_completion(
max_tokens=100, max_tokens=100,
seed=1, seed=1,
tools=tools, tools=tools,
@ -96,15 +102,15 @@ async def test_flash_llama_grammar_tools(flash_llama_grammar_tools, response_sna
) )
assert response.choices[0].message.content is None assert response.choices[0].message.content is None
assert response.choices[0].message.tool_calls == [ assert response.choices[0].message.tool_calls == [
{ ChatCompletionOutputToolCall(
"id": "0", id="0",
"type": "function", type="function",
"function": { function=ChatCompletionOutputFunctionDefinition(
"description": None, description=None,
"name": "get_current_weather", name="get_current_weather",
"arguments": {"format": "celsius", "location": "Brooklyn, New York"}, arguments='{"format":"fahrenheit","location":"Brooklyn, NY"}',
}, ),
} )
] ]
assert response == response_snapshot assert response == response_snapshot
@ -135,18 +141,25 @@ async def test_flash_llama_grammar_tools_openai(
) )
chunks = [] chunks = []
tool = ""
for chunk in stream: for chunk in stream:
tool += chunk.choices[0].delta.tool_calls[0].function.arguments
chunks.append(chunk) chunks.append(chunk)
assert (
tool
== '{"function": {"_name": "get_current_weather", "location": "Brooklyn, NY", "format": "fahrenheit"}}<|eot_id|>'
)
assert chunks == response_snapshot assert chunks == response_snapshot
@pytest.mark.asyncio @pytest.mark.asyncio
@pytest.mark.private @pytest.mark.private
async def test_flash_llama_grammar_tools_auto( async def test_flash_llama_grammar_tools_auto_nostream(
flash_llama_grammar_tools, response_snapshot flash_llama_grammar_tools, response_snapshot
): ):
response = await flash_llama_grammar_tools.chat( client = InferenceClient(base_url=f"{flash_llama_grammar_tools.base_url}/v1")
response = client.chat_completion(
max_tokens=100, max_tokens=100,
seed=1, seed=1,
tools=tools, tools=tools,
@ -165,15 +178,15 @@ async def test_flash_llama_grammar_tools_auto(
) )
assert response.choices[0].message.content is None assert response.choices[0].message.content is None
assert response.choices[0].message.tool_calls == [ assert response.choices[0].message.tool_calls == [
{ ChatCompletionOutputToolCall(
"id": "0", id="0",
"type": "function", type="function",
"function": { function=ChatCompletionOutputFunctionDefinition(
"description": None, description=None,
"name": "get_current_weather", name="get_current_weather",
"arguments": {"format": "celsius", "location": "Brooklyn, New York"}, arguments='{"format":"fahrenheit","location":"Brooklyn, NY"}',
}, ),
} )
] ]
assert response == response_snapshot assert response == response_snapshot
@ -181,10 +194,11 @@ async def test_flash_llama_grammar_tools_auto(
@pytest.mark.asyncio @pytest.mark.asyncio
@pytest.mark.private @pytest.mark.private
async def test_flash_llama_grammar_tools_choice( async def test_flash_llama_grammar_tools_choice_nostream(
flash_llama_grammar_tools, response_snapshot flash_llama_grammar_tools, response_snapshot
): ):
response = await flash_llama_grammar_tools.chat( client = InferenceClient(base_url=f"{flash_llama_grammar_tools.base_url}/v1")
response = client.chat_completion(
max_tokens=100, max_tokens=100,
seed=1, seed=1,
tools=tools, tools=tools,
@ -203,15 +217,15 @@ async def test_flash_llama_grammar_tools_choice(
) )
assert response.choices[0].message.content is None assert response.choices[0].message.content is None
assert response.choices[0].message.tool_calls == [ assert response.choices[0].message.tool_calls == [
{ ChatCompletionOutputToolCall(
"id": "0", id="0",
"type": "function", type="function",
"function": { function=ChatCompletionOutputFunctionDefinition(
"description": None, description=None,
"name": "get_current_weather", name="get_current_weather",
"arguments": {"format": "celsius", "location": "Brooklyn, New York"}, arguments='{"format":"fahrenheit","location":"Brooklyn, NY"}',
}, ),
} )
] ]
assert response == response_snapshot assert response == response_snapshot
@ -219,10 +233,11 @@ async def test_flash_llama_grammar_tools_choice(
@pytest.mark.asyncio @pytest.mark.asyncio
@pytest.mark.private @pytest.mark.private
async def test_flash_llama_grammar_tools_stream( async def test_flash_llama_grammar_tools_choice_stream(
flash_llama_grammar_tools, response_snapshot flash_llama_grammar_tools, response_snapshot
): ):
responses = await flash_llama_grammar_tools.chat( client = InferenceClient(base_url=f"{flash_llama_grammar_tools.base_url}/v1")
stream = client.chat_completion(
max_tokens=100, max_tokens=100,
seed=1, seed=1,
tools=tools, tools=tools,
@ -241,31 +256,27 @@ async def test_flash_llama_grammar_tools_stream(
stream=True, stream=True,
) )
count = 0
tool_calls_generated = "" tool_calls_generated = ""
last_response = None chunks = []
async for response in responses: for chunk in stream:
count += 1 tool_calls_generated += chunk.choices[0].delta.tool_calls[0].function.arguments
tool_calls_generated += ( assert chunk.choices[0].delta.content is None
response.choices[0].delta.tool_calls[0].function.arguments chunks.append(chunk)
)
last_response = response
assert response.choices[0].delta.content is None
assert ( assert (
tool_calls_generated tool_calls_generated
== '{"function": {"_name": "get_current_weather", "location": "Paris, France", "format": "celsius"}}<|eot_id|>' == '{"function": {"_name": "get_current_weather", "location": "Paris, France", "format": "celsius"}}<|eot_id|>'
) )
assert count == 28 assert chunks == response_snapshot
assert last_response == response_snapshot
@pytest.mark.asyncio @pytest.mark.asyncio
@pytest.mark.private @pytest.mark.private
async def test_flash_llama_grammar_tools_insufficient_information( async def test_flash_llama_grammar_tools_insufficient_information_nostream(
flash_llama_grammar_tools, response_snapshot flash_llama_grammar_tools, response_snapshot
): ):
responses = await flash_llama_grammar_tools.chat( client = InferenceClient(base_url=f"{flash_llama_grammar_tools.base_url}/v1")
response = client.chat_completion(
max_tokens=100, max_tokens=100,
seed=24, seed=24,
tools=tools, tools=tools,
@ -283,10 +294,13 @@ async def test_flash_llama_grammar_tools_insufficient_information(
stream=False, stream=False,
) )
assert responses.choices[0].message.tool_calls is None content_generated = response.choices[0].message.content
assert responses.choices[0].message.content == "I am an AI assistant" assert response.choices[0].message.tool_calls is None
assert responses == response_snapshot ######## FIXME before MERGE ############################
# TODO This is different from the streaming case, this is NOT normal.
assert content_generated == "I am a helpful assistant!"
assert response == response_snapshot
@pytest.mark.asyncio @pytest.mark.asyncio
@ -294,7 +308,8 @@ async def test_flash_llama_grammar_tools_insufficient_information(
async def test_flash_llama_grammar_tools_insufficient_information_stream( async def test_flash_llama_grammar_tools_insufficient_information_stream(
flash_llama_grammar_tools, response_snapshot flash_llama_grammar_tools, response_snapshot
): ):
responses = await flash_llama_grammar_tools.chat( client = InferenceClient(base_url=f"{flash_llama_grammar_tools.base_url}/v1")
stream = client.chat_completion(
max_tokens=100, max_tokens=100,
seed=24, seed=24,
tools=tools, tools=tools,
@ -312,26 +327,24 @@ async def test_flash_llama_grammar_tools_insufficient_information_stream(
stream=True, stream=True,
) )
count = 0
content_generated = "" content_generated = ""
last_response = None chunks = []
async for response in responses: for chunk in stream:
count += 1 content_generated += chunk.choices[0].delta.content
content_generated += response.choices[0].delta.content chunks.append(chunk)
last_response = response assert chunk.choices[0].delta.tool_calls is None
assert response.choices[0].delta.tool_calls is None
assert count == 5 assert content_generated == "I am a helpful assistant"
assert content_generated == "I am an AI assistant" assert chunks == response_snapshot
assert last_response == response_snapshot
@pytest.mark.asyncio @pytest.mark.asyncio
@pytest.mark.private @pytest.mark.private
async def test_flash_llama_grammar_tools_sea_creatures_stream( async def test_flash_llama_grammar_tools_sea_creatures_stream_auto(
flash_llama_grammar_tools, response_snapshot flash_llama_grammar_tools, response_snapshot
): ):
responses = await flash_llama_grammar_tools.chat( client = InferenceClient(base_url=f"{flash_llama_grammar_tools.base_url}/v1")
stream = client.chat_completion(
max_tokens=100, max_tokens=100,
seed=24, seed=24,
tools=tools, tools=tools,
@ -349,21 +362,18 @@ async def test_flash_llama_grammar_tools_sea_creatures_stream(
stream=True, stream=True,
) )
count = 0
content_generated = "" content_generated = ""
last_response = None chunks = []
async for response in responses: for chunk in stream:
count += 1 content_generated += chunk.choices[0].delta.content
content_generated += response.choices[0].delta.content chunks.append(chunk)
last_response = response assert chunk.choices[0].delta.tool_calls is None
assert response.choices[0].delta.tool_calls is None
assert count == 62
assert ( assert (
content_generated content_generated
== "Once upon a time, in the ocean, there lived three sea creatures. There was a wise old octopus named Bob, a mischievous seagull named Sam, and a gentle sea turtle named Luna. They all lived together in a beautiful coral reef, surrounded by colorful fish and swaying sea fans" == "There was a wise old octopus named Oracle. He lived in a cozy little cave beneath the waves with his best friend, a curious seahorse named Finley. One day, Finley met a playful dolphin named Daisy, and the three became inseparable. They spent their days exploring the ocean, playing hide-and-seek, and learning about the wonders of the sea from Oracle"
) )
assert last_response == response_snapshot assert chunks == response_snapshot
@pytest.mark.asyncio @pytest.mark.asyncio
@ -371,7 +381,8 @@ async def test_flash_llama_grammar_tools_sea_creatures_stream(
async def test_flash_llama_grammar_tools_sea_creatures_stream_required( async def test_flash_llama_grammar_tools_sea_creatures_stream_required(
flash_llama_grammar_tools, response_snapshot flash_llama_grammar_tools, response_snapshot
): ):
responses = await flash_llama_grammar_tools.chat( client = InferenceClient(base_url=f"{flash_llama_grammar_tools.base_url}/v1")
stream = client.chat_completion(
max_tokens=100, max_tokens=100,
seed=24, seed=24,
tools=tools, tools=tools,
@ -389,23 +400,17 @@ async def test_flash_llama_grammar_tools_sea_creatures_stream_required(
stream=True, stream=True,
) )
count = 0
tool_calls_generated = "" tool_calls_generated = ""
last_response = None chunks = []
async for response in responses: for chunk in stream:
count += 1 assert chunk.choices[0].delta.content is None
assert response.choices[0].delta.content is None tool_calls_generated += chunk.choices[0].delta.tool_calls[0].function.arguments
tool_calls_generated += (
response.choices[0].delta.tool_calls[0].function.arguments
)
last_response = response
assert count == 29
assert ( assert (
tool_calls_generated tool_calls_generated
== '{"function": {"_name": "get_current_weather", "location": "San Francisco, CA", "format": "celsius"}}<|eot_id|>' == '{"function": {"_name": "get_n_day_weather_forecast", "location": "San Francisco, CA", "format": "fahrenheit", "num_days":3}}<|eot_id|>'
) )
assert last_response == response_snapshot assert chunks == response_snapshot
@pytest.mark.asyncio @pytest.mark.asyncio
@ -413,7 +418,8 @@ async def test_flash_llama_grammar_tools_sea_creatures_stream_required(
async def test_flash_llama_grammar_tools_sea_creatures_stream_none( async def test_flash_llama_grammar_tools_sea_creatures_stream_none(
flash_llama_grammar_tools, response_snapshot flash_llama_grammar_tools, response_snapshot
): ):
responses = await flash_llama_grammar_tools.chat( client = InferenceClient(base_url=f"{flash_llama_grammar_tools.base_url}/v1")
stream = client.chat_completion(
max_tokens=100, max_tokens=100,
seed=24, seed=24,
tools=tools, tools=tools,
@ -431,22 +437,18 @@ async def test_flash_llama_grammar_tools_sea_creatures_stream_none(
stream=True, stream=True,
) )
count = 0
content_generated = "" content_generated = ""
last_response = None chunks = []
async for response in responses: for chunk in stream:
count += 1 chunks.append(chunk)
content_generated += response.choices[0].delta.content content_generated += chunk.choices[0].delta.content
last_response = response assert chunk.choices[0].delta.tool_calls is None
assert response.choices[0].delta.tool_calls is None
assert count == 100
print(content_generated)
assert ( assert (
content_generated content_generated
== "Once upon a time, in a vibrant ocean filled with coral reefs and schools of shimmering fish, lived three dear friends: Luna the sea turtle, Finley the friendly fish, and Crusty the wise crab.\n\nLuna was the oldest of the three. She had traveled the world, exploring hidden caves and shipwrecks, and collecting sparkling shells and shiny pebbles. Her shell was a beautiful mosaic of blues and greens, and her gentle eyes twinkled with the secrets of the deep" == "Once upon a time, in a vibrant ocean filled with coral reefs and schools of shimmering fish, lived three dear friends: Luna the sea turtle, Finley the friendly fish, and Crusty the wise crab.\n\nLuna was the oldest of the three. She had traveled the world, exploring hidden caves and shipwrecks, and collecting sparkling shells and shiny pebbles. Her shell was a beautiful mosaic of blues and greens, and her gentle eyes twinkled with the secrets of the deep"
) )
assert last_response == response_snapshot assert chunks == response_snapshot
@pytest.mark.asyncio @pytest.mark.asyncio
@ -454,57 +456,37 @@ async def test_flash_llama_grammar_tools_sea_creatures_stream_none(
async def test_flash_llama_grammar_tools_sea_creatures_stream_function_object( async def test_flash_llama_grammar_tools_sea_creatures_stream_function_object(
flash_llama_grammar_tools, response_snapshot flash_llama_grammar_tools, response_snapshot
): ):
# using `requests` to send the request until the client library supports tool_choice as a function object client = InferenceClient(base_url=f"{flash_llama_grammar_tools.base_url}/v1")
responses = requests.post( stream = client.chat_completion(
f"{flash_llama_grammar_tools.base_url}/v1/chat/completions", messages=[
headers=flash_llama_grammar_tools.headers, {
json={ "role": "system",
"model": "tgi", "content": "You're a helpful assistant! Answer the users question best you can. If the question is not answerable by the tools, just generate a response.",
"messages": [
{
"role": "system",
"content": "You're a helpful assistant! Answer the users question best you can. If the question is not answerable by the tools, just generate a response.",
},
{
"role": "user",
"content": "Tell me a story about 3 sea creatures",
},
],
"tools": tools,
"tool_choice": {
"type": "function",
"function": {"name": "get_n_day_weather_forecast"},
}, },
"seed": 24, {
"max_tokens": 100, "role": "user",
"stream": True, "content": "Tell me a story about 3 sea creatures",
},
],
tools=tools,
tool_choice={
"type": "function",
"function": {"name": "get_n_day_weather_forecast"},
}, },
max_tokens=100,
seed=24,
stream=True, stream=True,
) )
# iterate over the response in chunks chunks = []
count = 0
tool_calls_generated = "" tool_calls_generated = ""
last_response = None for chunk in stream:
for chunk in responses.iter_content(chunk_size=1024): tool_calls_generated += chunk.choices[0].delta.tool_calls[0].function.arguments
if chunk: chunks.append(chunk)
count += 1
# remove the "data: " prefix, trailing newline, and split the chunk into individual lines
lines = chunk.decode("utf-8").replace("data: ", "").rstrip("\n").split("\n")
for line in lines:
if line == "[DONE]":
break
response = json.loads(line)
tool_calls_generated += response["choices"][0]["delta"]["tool_calls"][
0
]["function"]["arguments"]
last_response = response
assert count == 39
assert ( assert (
tool_calls_generated tool_calls_generated
== '{"function": {"_name": "get_n_day_weather_forecast", "location": "San Francisco, CA", "format": "celsius", "num_days":3}}<|eot_id|>' == '{"function": {"_name": "get_n_day_weather_forecast", "location": "San Francisco, CA", "format": "celsius", "num_days": 3}}<|eot_id|>'
) )
assert last_response == response_snapshot assert chunks == response_snapshot
@pytest.mark.asyncio @pytest.mark.asyncio
@ -512,7 +494,8 @@ async def test_flash_llama_grammar_tools_sea_creatures_stream_function_object(
async def test_flash_llama_tool_reply_response( async def test_flash_llama_tool_reply_response(
flash_llama_grammar_tools, response_snapshot flash_llama_grammar_tools, response_snapshot
): ):
responses = await flash_llama_grammar_tools.chat( client = InferenceClient(base_url=f"{flash_llama_grammar_tools.base_url}/v1")
response = client.chat_completion(
max_tokens=100, max_tokens=100,
seed=42, seed=42,
messages=[ messages=[
@ -536,10 +519,10 @@ async def test_flash_llama_tool_reply_response(
stream=False, stream=False,
) )
assert responses.choices[0].message.tool_calls is None assert response.choices[0].message.tool_calls is None
assert ( assert (
responses.choices[0].message.content response.choices[0].message.content
== "I can't access real-time data, but I can provide you with current conditions and forecast for Paris, France:\n\nThe current conditions in Paris are mostly cloudy with a temperature of 6.7°C (44.1°F). \n\nPlease note that the actual weather may differ from this information, and I recommend checking the forecast on a reliable weather website for the most up-to-date information." == "I can't access real-time data, but I can provide you with current conditions and forecast for Paris, France:\n\nThe current conditions in Paris are mostly cloudy with a temperature of 6.7°C (44.1°F). \n\nPlease note that the actual weather may differ from the provided information. For up-to-date information, I suggest checking a reliable weather website or app for the latest conditions and forecast."
) )
assert responses == response_snapshot assert response == response_snapshot

1
integration-tests/pyproject.toml
View File

@ -14,6 +14,7 @@ dependencies = [
"docker>=7", "docker>=7",
"numpy>=2.0", "numpy>=2.0",
"openai>=1.65", "openai>=1.65",
"huggingface_hub>=0.29",
] ]
[tool.isort] [tool.isort]

4
integration-tests/requirements.txt
View File

@ -40,7 +40,9 @@ httpcore==1.0.7
httpx==0.28.1 httpx==0.28.1
# via openai # via openai
huggingface-hub==0.29.0 huggingface-hub==0.29.0
# via text-generation # via
# text-generation-integration-tests (pyproject.toml)
# text-generation
idna==3.10 idna==3.10
# via # via
# anyio # anyio

4
router/src/infer/chat_template.rs
View File

@ -1189,7 +1189,7 @@ TOOL CALL ID: 0
let tool_prompt = "This default prompt will be used".to_string(); let tool_prompt = "This default prompt will be used".to_string();
let tools_and_prompt = Some((tools, tool_prompt)); let tools_and_prompt = Some((tools, tool_prompt));
let result = ct.apply(msgs, tools_and_prompt); let result = ct.apply(msgs, tools_and_prompt);
let expected = "<s>[INST] I'd like to show off how chat templating works! [/INST]Great! How can I help you today?</s> [INST] Just testing\n---\n[{\"type\":\"function\",\"function\":{\"description\":\"Get the current weather\",\"name\":\"get_current_weather\",\"arguments\":{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\",\"description\":\"The city and state, e.g. San Francisco, CA\"},\"format\":{\"type\":\"string\",\"enum\":[\"celsius\",\"fahrenheit\"],\"description\":\"The temperature unit to use. Infer this from the users location.\"}},\"required\":[\"location\",\"format\"]}}}]\nThis default prompt will be used [/INST]".to_string(); let expected = "<s>[INST] I'd like to show off how chat templating works! [/INST]Great! How can I help you today?</s> [INST] Just testing\n---\n[{\"type\":\"function\",\"function\":{\"description\":\"Get the current weather\",\"name\":\"get_current_weather\",\"arguments\":\"{\\\"type\\\":\\\"object\\\",\\\"properties\\\":{\\\"location\\\":{\\\"type\\\":\\\"string\\\",\\\"description\\\":\\\"The city and state, e.g. San Francisco, CA\\\"},\\\"format\\\":{\\\"type\\\":\\\"string\\\",\\\"enum\\\":[\\\"celsius\\\",\\\"fahrenheit\\\"],\\\"description\\\":\\\"The temperature unit to use. Infer this from the users location.\\\"}},\\\"required\\\":[\\\"location\\\",\\\"format\\\"]}\"}}]\nThis default prompt will be used [/INST]".to_string();
assert_eq!(result.unwrap(), expected); assert_eq!(result.unwrap(), expected);
} }
@ -1227,7 +1227,7 @@ TOOL CALL ID: 0
let tool_prompt = "This default prompt will be used".to_string(); let tool_prompt = "This default prompt will be used".to_string();
let tools_and_prompt = Some((tools, tool_prompt)); let tools_and_prompt = Some((tools, tool_prompt));
let result = ct.apply(msgs, tools_and_prompt); let result = ct.apply(msgs, tools_and_prompt);
let expected = "<s><|start_header_id|>system<|end_header_id|>\n\nEnvironment: ipython\nCutting Knowledge Date: December 2023\nToday Date: 26 Jul 2024\n\nYoure a helpful assistant! Answer the users question best you can.<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nGiven the following functions, please respond with a JSON for a function call with its proper arguments that best answers the given prompt.\n\nRespond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.Do not use variables.\n\n{\n \"function\": {\n \"arguments\": {\n \"properties\": {\n \"format\": {\n \"description\": \"The temperature unit to use. Infer this from the users location.\",\n \"enum\": [\n \"celsius\",\n \"fahrenheit\"\n ],\n \"type\": \"string\"\n },\n \"location\": {\n \"description\": \"The city and state, e.g. San Francisco, CA\",\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"location\",\n \"format\"\n ],\n \"type\": \"object\"\n },\n \"description\": \"Get the current weather\",\n \"name\": \"get_current_weather\"\n },\n \"type\": \"function\"\n}\n\nWhat is the weather like in Brooklyn, New York?\n---\nThis default prompt will be used<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n".to_string(); let expected = "<s><|start_header_id|>system<|end_header_id|>\n\nEnvironment: ipython\nCutting Knowledge Date: December 2023\nToday Date: 26 Jul 2024\n\nYoure a helpful assistant! Answer the users question best you can.<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nGiven the following functions, please respond with a JSON for a function call with its proper arguments that best answers the given prompt.\n\nRespond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.Do not use variables.\n\n{\n \"function\": {\n \"arguments\": \"{\\\"type\\\":\\\"object\\\",\\\"properties\\\":{\\\"location\\\":{\\\"type\\\":\\\"string\\\",\\\"description\\\":\\\"The city and state, e.g. San Francisco, CA\\\"},\\\"format\\\":{\\\"type\\\":\\\"string\\\",\\\"enum\\\":[\\\"celsius\\\",\\\"fahrenheit\\\"],\\\"description\\\":\\\"The temperature unit to use. Infer this from the users location.\\\"}},\\\"required\\\":[\\\"location\\\",\\\"format\\\"]}\",\n \"description\": \"Get the current weather\",\n \"name\": \"get_current_weather\"\n },\n \"type\": \"function\"\n}\n\nWhat is the weather like in Brooklyn, New York?\n---\nThis default prompt will be used<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n".to_string();
assert_eq!(result.unwrap(), expected); assert_eq!(result.unwrap(), expected);
} }
} }

11
router/src/lib.rs
View File

@ -1138,10 +1138,17 @@ pub struct FunctionDefinition {
#[serde(default)] #[serde(default)]
pub description: Option<String>, pub description: Option<String>,
pub name: String, pub name: String,
#[serde(alias = "parameters")] #[serde(alias = "parameters", serialize_with = "serialize_as_string")]
pub arguments: serde_json::Value, pub arguments: serde_json::Value,
} }
fn serialize_as_string<S>(value: &serde_json::Value, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
serializer.serialize_str(&value.to_string())
}
#[derive(Clone, Debug, Deserialize, Serialize, ToSchema)] #[derive(Clone, Debug, Deserialize, Serialize, ToSchema)]
#[cfg_attr(test, derive(PartialEq))] #[cfg_attr(test, derive(PartialEq))]
pub(crate) struct Tool { pub(crate) struct Tool {
@ -1730,7 +1737,7 @@ mod tests {
let serialized = serde_json::to_string(&message).unwrap(); let serialized = serde_json::to_string(&message).unwrap();
assert_eq!( assert_eq!(
serialized, serialized,
r#"{"role":"assistant","tool_calls":[{"id":"0","type":"function","function":{"description":null,"name":"myfn","arguments":{"format":"csv"}}}]}"# r#"{"role":"assistant","tool_calls":[{"id":"0","type":"function","function":{"description":null,"name":"myfn","arguments":"{\"format\":\"csv\"}"}}]}"#
); );
} }