mirror of
https://github.com/huggingface/text-generation-inference.git
synced 2025-04-21 23:12:07 +00:00
a6a0c97ed9
* wip * rollback * refactor to use prefix/postfix namming + fix all_input_ids_tensor * maybe patching vlms? * fix filter and concat * wip, no filter, no concat * current * add prepare_for_prefill * working * load tested * re-create slots * re-create slots * fix slot_filtering_indices * feedback loop * remove log * fix benchmarker * fix vlm and seq2seq * rename to cache and input lengths * fix prefill logprobs * fix launcher * fix logprobs? * idk at this point * max input length * omfg * remove debugging lines * fix tests * fix mllama * fix cargo tests * remove support chunking for paged * Fixing non blocked attentions * Fixing dtype + AMD, Ipex targets. * lint fix. * rename * Fix prefix_caching variable, remove defaults in server (confusing a lot of the times). * Add simple resolution when user specifies ATTENTION=paged. * Put back non default simple tests. * Fix env name --------- Co-authored-by: Nicolas Patry <patry.nicolas@protonmail.com>
103 lines
3.0 KiB
Python
103 lines
3.0 KiB
Python
import pytest
|
|
import requests
|
|
from pydantic import BaseModel
|
|
from typing import List
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def llama_grammar_handle(launcher):
|
|
with launcher(
|
|
"TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
|
num_shard=1,
|
|
disable_grammar_support=False,
|
|
use_flash_attention=False,
|
|
max_batch_prefill_tokens=3000,
|
|
) as handle:
|
|
yield handle
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
async def llama_grammar(llama_grammar_handle):
|
|
await llama_grammar_handle.health(300)
|
|
return llama_grammar_handle.client
|
|
|
|
|
|
@pytest.mark.release
|
|
@pytest.mark.asyncio
|
|
async def test_grammar_response_format_llama_json(llama_grammar, response_snapshot):
|
|
class Weather(BaseModel):
|
|
unit: str
|
|
temperature: List[int]
|
|
|
|
# send the request
|
|
response = requests.post(
|
|
f"{llama_grammar.base_url}/v1/chat/completions",
|
|
headers=llama_grammar.headers,
|
|
json={
|
|
"model": "tgi",
|
|
"messages": [
|
|
{
|
|
"role": "system",
|
|
"content": f"Respond to the users questions and answer them in the following format: {Weather.schema()}",
|
|
},
|
|
{
|
|
"role": "user",
|
|
"content": "What's the weather like the next 3 days in San Francisco, CA?",
|
|
},
|
|
],
|
|
"seed": 42,
|
|
"max_tokens": 500,
|
|
"response_format": {"type": "json_object", "value": Weather.schema()},
|
|
},
|
|
)
|
|
|
|
chat_completion = response.json()
|
|
called = chat_completion["choices"][0]["message"]["content"]
|
|
|
|
assert response.status_code == 200
|
|
assert (
|
|
called
|
|
== '{\n "temperature": [\n 35,\n 34,\n 36\n ],\n "unit": "°c"\n}'
|
|
)
|
|
assert chat_completion == response_snapshot
|
|
|
|
|
|
@pytest.mark.release
|
|
@pytest.mark.asyncio
|
|
async def test_grammar_response_format_llama_error_if_tools_not_installed(
|
|
llama_grammar,
|
|
):
|
|
class Weather(BaseModel):
|
|
unit: str
|
|
temperature: List[int]
|
|
|
|
# send the request
|
|
response = requests.post(
|
|
f"{llama_grammar.base_url}/v1/chat/completions",
|
|
headers=llama_grammar.headers,
|
|
json={
|
|
"model": "tgi",
|
|
"messages": [
|
|
{
|
|
"role": "system",
|
|
"content": f"Respond to the users questions and answer them in the following format: {Weather.schema()}",
|
|
},
|
|
{
|
|
"role": "user",
|
|
"content": "What's the weather like the next 3 days in San Francisco, CA?",
|
|
},
|
|
],
|
|
"seed": 42,
|
|
"max_tokens": 500,
|
|
"tools": [],
|
|
"response_format": {"type": "json_object", "value": Weather.schema()},
|
|
},
|
|
)
|
|
|
|
# 422 means the server was unable to process the request because it contains invalid data.
|
|
assert response.status_code == 422
|
|
assert response.json() == {
|
|
"error": "Tool error: Grammar and tools are mutually exclusive",
|
|
"error_type": "tool_error",
|
|
}
|