huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-10 11:54:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Getting closer to the non gptq test (stop sequence doesn't work).

Browse Source
This commit is contained in:
Nicolas Patry 2023-07-21 08:15:25 +00:00
parent 8b6a262539
commit afb39404e1
4 changed files with 230 additions and 316 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

89
integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq.json
View File

@ -10,94 +10,79 @@
"text": "<s>" "text": "<s>"
}, },
{ {
"id": 20628, "id": 4321,
"logprob": -10.4296875, "logprob": -9.59375,
"text": "Today" "text": "Test"
}, },
{ {
"id": 306, "id": 2009,
"logprob": -2.4140625, "logprob": -9.6640625,
"text": "I" "text": "request"
},
{
"id": 626,
"logprob": -1.8818359,
"text": "am"
},
{
"id": 297,
"logprob": -4.4804688,
"text": "in"
},
{
"id": 3444,
"logprob": -7.0820312,
"text": "France"
} }
], ],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
"id": 29892, "id": 29918,
"logprob": -1.2949219, "logprob": -2.3867188,
"special": false, "special": false,
"text": "," "text": "_"
}, },
{ {
"id": 297, "id": 5338,
"logprob": -1.9414062, "logprob": -2.8183594,
"special": false, "special": false,
"text": " in" "text": "uri"
}, },
{ {
"id": 278, "id": 13,
"logprob": -0.75390625, "logprob": -1.6367188,
"special": false, "special": false,
"text": " the" "text": "\n"
}, },
{ {
"id": 7062, "id": 3057,
"logprob": -2.9101562, "logprob": -1.0527344,
"special": false, "special": false,
"text": " south" "text": "Test"
}, },
{ {
"id": 310, "id": 2009,
"logprob": -1.0263672, "logprob": -0.6542969,
"special": false, "special": false,
"text": " of" "text": " request"
}, },
{ {
"id": 278, "id": 29918,
"logprob": -0.5751953, "logprob": -0.056121826,
"special": false, "special": false,
"text": " the" "text": "_"
}, },
{ {
"id": 4234, "id": 5338,
"logprob": -0.30273438, "logprob": -0.01600647,
"special": false, "special": false,
"text": " country" "text": "uri"
}, },
{ {
"id": 29892, "id": 13,
"logprob": -0.69091797, "logprob": -0.87939453,
"special": false, "special": false,
"text": "," "text": "\n"
}, },
{ {
"id": 297, "id": 3057,
"logprob": -1.1015625, "logprob": -0.7529297,
"special": false, "special": false,
"text": " in" "text": "Test"
}, },
{ {
"id": 278, "id": 2009,
"logprob": -0.5175781, "logprob": -0.2980957,
"special": false, "special": false,
"text": " the" "text": " request"
} }
] ]
}, },
"generated_text": ", in the south of the country, in the" "generated_text": "_uri\nTest request_uri\nTest request"
} }

94
integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq_all_params.json
View File

@ -10,89 +10,79 @@
"text": "<s>" "text": "<s>"
}, },
{ {
"id": 4272, "id": 4321,
"logprob": -12.390625, "logprob": -9.6015625,
"text": "city" "text": "Test"
}, },
{ {
"id": 310, "id": 2009,
"logprob": -2.5292969, "logprob": -9.6640625,
"text": "of" "text": "request"
},
{
"id": 3444,
"logprob": -11.25,
"text": "France"
},
{
"id": 338,
"logprob": -4.953125,
"text": "is"
} }
], ],
"seed": 0, "seed": 0,
"tokens": [ "tokens": [
{ {
"id": 278, "id": 29899,
"logprob": -0.1796875, "logprob": -1.1640625,
"special": false, "special": false,
"text": " the" "text": "-"
}, },
{ {
"id": 12949, "id": 1454,
"logprob": -2.2792969, "logprob": -0.07543945,
"special": false, "special": false,
"text": " seat" "text": "for"
}, },
{ {
"id": 310, "id": 29899,
"logprob": 0.0, "logprob": 0.0,
"special": false, "special": false,
"text": " of" "text": "-"
}, },
{ {
"id": 263, "id": 9342,
"logprob": -0.09301758,
"special": false,
"text": " a"
},
{
"id": 5917,
"logprob": -1.3974609,
"special": false,
"text": " Roman"
},
{
"id": 11865,
"logprob": 0.0, "logprob": 0.0,
"special": false, "special": false,
"text": " Catholic" "text": "comment"
}, },
{ {
"id": 3190, "id": 29901,
"logprob": 0.0, "logprob": 0.0,
"special": false, "special": false,
"text": " arch" "text": ":"
}, },
{ {
"id": 28693, "id": 396,
"logprob": -0.2956543,
"special": false,
"text": " #"
},
{
"id": 29906,
"logprob": -0.52734375,
"special": false,
"text": "2"
},
{
"id": 29900,
"logprob": -0.6899414,
"special": false,
"text": "0"
},
{
"id": 29896,
"logprob": 0.0, "logprob": 0.0,
"special": false, "special": false,
"text": "bishop" "text": "1"
}, },
{ {
"id": 29892, "id": 29946,
"logprob": 0.0, "logprob": -1.5068359,
"special": false, "special": false,
"text": "," "text": "4"
},
{
"id": 1058,
"logprob": -0.9433594,
"special": false,
"text": " who"
} }
] ]
}, },
"generated_text": "The capital city of France isthe seat of a Roman Catholic archbishop, who" "generated_text": "Test request-for-comment: #2014"
} }

356
integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq_load.json
View File

@ -11,96 +11,81 @@
"text": "<s>" "text": "<s>"
}, },
{ {
"id": 20628, "id": 4321,
"logprob": -10.4296875, "logprob": -9.6015625,
"text": "Today" "text": "Test"
}, },
{ {
"id": 306, "id": 2009,
"logprob": -2.4179688, "logprob": -9.671875,
"text": "I" "text": "request"
},
{
"id": 626,
"logprob": -1.8876953,
"text": "am"
},
{
"id": 297,
"logprob": -4.484375,
"text": "in"
},
{
"id": 3444,
"logprob": -7.0820312,
"text": "France"
} }
], ],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
"id": 29892, "id": 29918,
"logprob": -1.2958984, "logprob": -2.3828125,
"special": false, "special": false,
"text": "," "text": "_"
}, },
{ {
"id": 297, "id": 5338,
"logprob": -1.9423828, "logprob": -2.8105469,
"special": false, "special": false,
"text": " in" "text": "uri"
}, },
{ {
"id": 278, "id": 13,
"logprob": -0.7475586, "logprob": -1.6396484,
"special": false, "special": false,
"text": " the" "text": "\n"
}, },
{ {
"id": 7062, "id": 3057,
"logprob": -2.9101562, "logprob": -1.0546875,
"special": false, "special": false,
"text": " south" "text": "Test"
}, },
{ {
"id": 310, "id": 2009,
"logprob": -1.0380859, "logprob": -0.6513672,
"special": false, "special": false,
"text": " of" "text": " request"
}, },
{ {
"id": 278, "id": 29918,
"logprob": -0.5761719, "logprob": -0.056365967,
"special": false, "special": false,
"text": " the" "text": "_"
}, },
{ {
"id": 4234, "id": 5338,
"logprob": -0.30297852, "logprob": -0.016082764,
"special": false, "special": false,
"text": " country" "text": "uri"
}, },
{ {
"id": 29892, "id": 13,
"logprob": -0.6933594, "logprob": -0.87841797,
"special": false, "special": false,
"text": "," "text": "\n"
}, },
{ {
"id": 297, "id": 3057,
"logprob": -1.0966797, "logprob": -0.7548828,
"special": false, "special": false,
"text": " in" "text": "Test"
}, },
{ {
"id": 278, "id": 2009,
"logprob": -0.51708984, "logprob": -0.29711914,
"special": false, "special": false,
"text": " the" "text": " request"
} }
] ]
}, },
"generated_text": ", in the south of the country, in the" "generated_text": "_uri\nTest request_uri\nTest request"
}, },
{ {
"details": { "details": {
@ -114,96 +99,81 @@
"text": "<s>" "text": "<s>"
}, },
{ {
"id": 20628, "id": 4321,
"logprob": -10.53125, "logprob": -9.6015625,
"text": "Today" "text": "Test"
}, },
{ {
"id": 306, "id": 2009,
"logprob": -2.4609375, "logprob": -9.6640625,
"text": "I" "text": "request"
},
{
"id": 626,
"logprob": -1.8857422,
"text": "am"
},
{
"id": 297,
"logprob": -4.484375,
"text": "in"
},
{
"id": 3444,
"logprob": -7.1015625,
"text": "France"
} }
], ],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
"id": 29892, "id": 29918,
"logprob": -1.2910156, "logprob": -2.3828125,
"special": false, "special": false,
"text": "," "text": "_"
}, },
{ {
"id": 297, "id": 5338,
"logprob": -1.9375, "logprob": -2.828125,
"special": false, "special": false,
"text": " in" "text": "uri"
}, },
{ {
"id": 278, "id": 13,
"logprob": -0.7416992, "logprob": -1.6386719,
"special": false, "special": false,
"text": " the" "text": "\n"
}, },
{ {
"id": 7062, "id": 3057,
"logprob": -2.90625, "logprob": -1.0527344,
"special": false, "special": false,
"text": " south" "text": "Test"
}, },
{ {
"id": 310, "id": 2009,
"logprob": -1.0439453, "logprob": -0.6542969,
"special": false, "special": false,
"text": " of" "text": " request"
}, },
{ {
"id": 278, "id": 29918,
"logprob": -0.5654297, "logprob": -0.055877686,
"special": false, "special": false,
"text": " the" "text": "_"
}, },
{ {
"id": 4234, "id": 5338,
"logprob": -0.3125, "logprob": -0.016021729,
"special": false, "special": false,
"text": " country" "text": "uri"
}, },
{ {
"id": 29892, "id": 13,
"logprob": -0.69384766, "logprob": -0.8769531,
"special": false, "special": false,
"text": "," "text": "\n"
}, },
{ {
"id": 297, "id": 3057,
"logprob": -1.0976562, "logprob": -0.7583008,
"special": false, "special": false,
"text": " in" "text": "Test"
}, },
{ {
"id": 278, "id": 2009,
"logprob": -0.51416016, "logprob": -0.29833984,
"special": false, "special": false,
"text": " the" "text": " request"
} }
] ]
}, },
"generated_text": ", in the south of the country, in the" "generated_text": "_uri\nTest request_uri\nTest request"
}, },
{ {
"details": { "details": {
@ -217,96 +187,81 @@
"text": "<s>" "text": "<s>"
}, },
{ {
"id": 20628, "id": 4321,
"logprob": -10.53125, "logprob": -9.6015625,
"text": "Today" "text": "Test"
}, },
{ {
"id": 306, "id": 2009,
"logprob": -2.4609375, "logprob": -9.671875,
"text": "I" "text": "request"
},
{
"id": 626,
"logprob": -1.8857422,
"text": "am"
},
{
"id": 297,
"logprob": -4.484375,
"text": "in"
},
{
"id": 3444,
"logprob": -7.1015625,
"text": "France"
} }
], ],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
"id": 29892, "id": 29918,
"logprob": -1.2910156, "logprob": -2.3847656,
"special": false, "special": false,
"text": "," "text": "_"
}, },
{ {
"id": 297, "id": 5338,
"logprob": -1.9384766, "logprob": -2.8144531,
"special": false, "special": false,
"text": " in" "text": "uri"
}, },
{ {
"id": 278, "id": 13,
"logprob": -0.7426758, "logprob": -1.6396484,
"special": false, "special": false,
"text": " the" "text": "\n"
}, },
{ {
"id": 7062, "id": 3057,
"logprob": -2.9042969, "logprob": -1.0527344,
"special": false, "special": false,
"text": " south" "text": "Test"
}, },
{ {
"id": 310, "id": 2009,
"logprob": -1.0439453, "logprob": -0.65478516,
"special": false, "special": false,
"text": " of" "text": " request"
}, },
{ {
"id": 278, "id": 29918,
"logprob": -0.56103516, "logprob": -0.056243896,
"special": false, "special": false,
"text": " the" "text": "_"
}, },
{ {
"id": 4234, "id": 5338,
"logprob": -0.31323242, "logprob": -0.016143799,
"special": false, "special": false,
"text": " country" "text": "uri"
}, },
{ {
"id": 29892, "id": 13,
"logprob": -0.6982422, "logprob": -0.8808594,
"special": false, "special": false,
"text": "," "text": "\n"
}, },
{ {
"id": 297, "id": 3057,
"logprob": -1.0976562, "logprob": -0.75341797,
"special": false, "special": false,
"text": " in" "text": "Test"
}, },
{ {
"id": 278, "id": 2009,
"logprob": -0.52001953, "logprob": -0.2956543,
"special": false, "special": false,
"text": " the" "text": " request"
} }
] ]
}, },
"generated_text": ", in the south of the country, in the" "generated_text": "_uri\nTest request_uri\nTest request"
}, },
{ {
"details": { "details": {
@ -320,95 +275,80 @@
"text": "<s>" "text": "<s>"
}, },
{ {
"id": 20628, "id": 4321,
"logprob": -10.53125, "logprob": -9.6015625,
"text": "Today" "text": "Test"
}, },
{ {
"id": 306, "id": 2009,
"logprob": -2.4609375, "logprob": -9.6640625,
"text": "I" "text": "request"
},
{
"id": 626,
"logprob": -1.8857422,
"text": "am"
},
{
"id": 297,
"logprob": -4.484375,
"text": "in"
},
{
"id": 3444,
"logprob": -7.1015625,
"text": "France"
} }
], ],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
"id": 29892, "id": 29918,
"logprob": -1.2910156, "logprob": -2.3769531,
"special": false, "special": false,
"text": "," "text": "_"
}, },
{ {
"id": 297, "id": 5338,
"logprob": -1.9394531, "logprob": -2.8183594,
"special": false, "special": false,
"text": " in" "text": "uri"
}, },
{ {
"id": 278, "id": 13,
"logprob": -0.74121094, "logprob": -1.6396484,
"special": false, "special": false,
"text": " the" "text": "\n"
}, },
{ {
"id": 7062, "id": 3057,
"logprob": -2.90625, "logprob": -1.0546875,
"special": false, "special": false,
"text": " south" "text": "Test"
}, },
{ {
"id": 310, "id": 2009,
"logprob": -1.0439453, "logprob": -0.65478516,
"special": false, "special": false,
"text": " of" "text": " request"
}, },
{ {
"id": 278, "id": 29918,
"logprob": -0.56591797, "logprob": -0.05557251,
"special": false, "special": false,
"text": " the" "text": "_"
}, },
{ {
"id": 4234, "id": 5338,
"logprob": -0.31713867, "logprob": -0.01612854,
"special": false, "special": false,
"text": " country" "text": "uri"
}, },
{ {
"id": 29892, "id": 13,
"logprob": -0.69140625, "logprob": -0.8730469,
"special": false, "special": false,
"text": "," "text": "\n"
}, },
{ {
"id": 297, "id": 3057,
"logprob": -1.0957031, "logprob": -0.7519531,
"special": false, "special": false,
"text": " in" "text": "Test"
}, },
{ {
"id": 278, "id": 2009,
"logprob": -0.52001953, "logprob": -0.29785156,
"special": false, "special": false,
"text": " the" "text": " request"
} }
] ]
}, },
"generated_text": ", in the south of the country, in the" "generated_text": "_uri\nTest request_uri\nTest request"
} }
] ]

7
integration-tests/models/test_flash_llama_gptq.py
View File

@ -1,4 +1,3 @@
import pytest import pytest
@ -18,7 +17,7 @@ async def flash_llama_gptq(flash_llama_gptq_handle):
@pytest.mark.private @pytest.mark.private
async def test_flash_llama_gptq(flash_llama_gptq, response_snapshot): async def test_flash_llama_gptq(flash_llama_gptq, response_snapshot):
response = await flash_llama_gptq.generate( response = await flash_llama_gptq.generate(
"Today I am in France", max_new_tokens=10, decoder_input_details=True "Test request", max_new_tokens=10, decoder_input_details=True
) )
assert response.details.generated_tokens == 10 assert response.details.generated_tokens == 10
@ -29,7 +28,7 @@ async def test_flash_llama_gptq(flash_llama_gptq, response_snapshot):
@pytest.mark.private @pytest.mark.private
async def test_flash_llama_gptq_all_params(flash_llama_gptq, response_snapshot): async def test_flash_llama_gptq_all_params(flash_llama_gptq, response_snapshot):
response = await flash_llama_gptq.generate( response = await flash_llama_gptq.generate(
"The capital city of France is", "Test request",
max_new_tokens=10, max_new_tokens=10,
repetition_penalty=1.2, repetition_penalty=1.2,
return_full_text=True, return_full_text=True,
@ -50,7 +49,7 @@ async def test_flash_llama_gptq_all_params(flash_llama_gptq, response_snapshot):
@pytest.mark.asyncio @pytest.mark.asyncio
@pytest.mark.private @pytest.mark.private
async def test_flash_llama_gptq_load(flash_llama_gptq, generate_load, response_snapshot): async def test_flash_llama_gptq_load(flash_llama_gptq, generate_load, response_snapshot):
responses = await generate_load(flash_llama_gptq, "Today I am in France", max_new_tokens=10, n=4) responses = await generate_load(flash_llama_gptq, "Test request", max_new_tokens=10, n=4)
assert len(responses) == 4 assert len(responses) == 4
assert all([r.generated_text == responses[0].generated_text for r in responses]) assert all([r.generated_text == responses[0].generated_text for r in responses])