diff --git a/integration-tests/models/__snapshots__/test_flash_gemma3/test_flash_gemma3_image_base64_rgb_jpg.json b/integration-tests/models/__snapshots__/test_flash_gemma3/test_flash_gemma3_image_base64_rgb_jpg.json index ae67e006..41eb19fd 100644 --- a/integration-tests/models/__snapshots__/test_flash_gemma3/test_flash_gemma3_image_base64_rgb_jpg.json +++ b/integration-tests/models/__snapshots__/test_flash_gemma3/test_flash_gemma3_image_base64_rgb_jpg.json @@ -5,7 +5,7 @@ "index": 0, "logprobs": null, "message": { - "content": "Okay, let's analyze the image.\n\nThe image is a solid, bright white color. There is nothing else visible within it. \n\nIt's essentially a blank white canvas or a completely white square. \n\nIs there anything specific you'd like me to do with this image, such as describe it further or imagine what it might represent?", + "content": "Okay, let's analyze the image.\n\nThe image is a solid, bright white color. There is nothing else visible within it. \n\nIt's essentially a blank white square or rectangle.", "name": null, "role": "assistant", "tool_calls": null @@ -13,14 +13,14 @@ "usage": null } ], - "created": 1741965894, + "created": 1747062956, "id": "", "model": "google/gemma-3-4b-it", "object": "chat.completion", - "system_fingerprint": "3.2.1-dev0-native", + "system_fingerprint": "3.3.0-dev0-native", "usage": { - "completion_tokens": 74, + "completion_tokens": 42, "prompt_tokens": 277, - "total_tokens": 351 + "total_tokens": 319 } } diff --git a/integration-tests/models/__snapshots__/test_flash_gemma3/test_flash_gemma3_image_base64_rgb_png.json b/integration-tests/models/__snapshots__/test_flash_gemma3/test_flash_gemma3_image_base64_rgb_png.json index 786ced6c..1f3e2b91 100644 --- a/integration-tests/models/__snapshots__/test_flash_gemma3/test_flash_gemma3_image_base64_rgb_png.json +++ b/integration-tests/models/__snapshots__/test_flash_gemma3/test_flash_gemma3_image_base64_rgb_png.json @@ -1,11 +1,11 @@ { "choices": [ { - "finish_reason": "length", + "finish_reason": "stop", "index": 0, "logprobs": null, "message": { - "content": "Okay, let's analyze the image. \n\nThe image is entirely white, with a very subtle, faint outline of a stylized, cartoonish figure. It appears to be a simplified depiction of a person, likely a child, with a wide-eyed expression and a small, rounded body. \n\nIt's almost like a minimalist, iconic representation. \n\nDo you want me to try and describe it in more detail, or perhaps suggest what this image might represent (e.g", + "content": "Okay, let's analyze the image. \n\nThe image is a very plain, solid white square. That's it! \n\nIt's essentially a blank canvas. \n\nDo you want me to describe it in more detail, or are you interested in something else regarding this image?", "name": null, "role": "assistant", "tool_calls": null @@ -13,14 +13,14 @@ "usage": null } ], - "created": 1744396706, + "created": 1747062955, "id": "", "model": "google/gemma-3-4b-it", "object": "chat.completion", "system_fingerprint": "3.3.0-dev0-native", "usage": { - "completion_tokens": 100, + "completion_tokens": 62, "prompt_tokens": 277, - "total_tokens": 377 + "total_tokens": 339 } } diff --git a/integration-tests/models/__snapshots__/test_flash_gemma3/test_flash_gemma3_image_base64_rgba.json b/integration-tests/models/__snapshots__/test_flash_gemma3/test_flash_gemma3_image_base64_rgba.json index 571478ee..42a2be01 100644 --- a/integration-tests/models/__snapshots__/test_flash_gemma3/test_flash_gemma3_image_base64_rgba.json +++ b/integration-tests/models/__snapshots__/test_flash_gemma3/test_flash_gemma3_image_base64_rgba.json @@ -5,7 +5,7 @@ "index": 0, "logprobs": null, "message": { - "content": "Okay, let's analyze the image. \n\nThe transparent image reveals a stylized depiction of **a human head**. It's a minimalist, geometric representation, showing the basic shapes of the skull, eye sockets, and head outline. \n\nIf you'd like, you can give me more details about the image or ask me to focus on a specific aspect of it.", + "content": "Okay, let's analyze the image. \n\nThe transparent image reveals a stylized depiction of **a human head**. It's a minimalist, geometric representation, showing the basic shapes of the skull, eye sockets, and head outline. \n\nDo you want me to describe any specific element of the image in more detail?", "name": null, "role": "assistant", "tool_calls": null @@ -13,14 +13,14 @@ "usage": null } ], - "created": 1744396703, + "created": 1747062952, "id": "", "model": "google/gemma-3-4b-it", "object": "chat.completion", "system_fingerprint": "3.3.0-dev0-native", "usage": { - "completion_tokens": 78, + "completion_tokens": 67, "prompt_tokens": 277, - "total_tokens": 355 + "total_tokens": 344 } } diff --git a/integration-tests/models/__snapshots__/test_flash_gemma3/test_flash_gemma3_image_cow.json b/integration-tests/models/__snapshots__/test_flash_gemma3/test_flash_gemma3_image_cow.json index 9fb0c4c5..c1a4b402 100644 --- a/integration-tests/models/__snapshots__/test_flash_gemma3/test_flash_gemma3_image_cow.json +++ b/integration-tests/models/__snapshots__/test_flash_gemma3/test_flash_gemma3_image_cow.json @@ -5,7 +5,7 @@ "index": 0, "logprobs": null, "message": { - "content": "Here's a description of what's shown in the image:\n\nThe image depicts a brown cow standing on a sandy beach. The beach has turquoise water and a distant island visible in the background. The sky is bright blue with some white clouds. \n\nIt's a quite a humorous and unusual scene – a cow enjoying a day at the beach!", + "content": "Here's a description of what's shown in the image:\n\nThe image depicts a brown cow standing on a sandy beach. The cow has white markings on its face and ears. Behind the cow is a clear blue ocean with a distant island, and a bright blue sky with some scattered clouds. \n\nIt's a rather charming and unexpected sight – a cow enjoying a beach day!", "name": null, "role": "assistant", "tool_calls": null @@ -13,14 +13,14 @@ "usage": null } ], - "created": 1744396699, + "created": 1747123513, "id": "", "model": "google/gemma-3-4b-it", "object": "chat.completion", "system_fingerprint": "3.3.0-dev0-native", "usage": { - "completion_tokens": 74, + "completion_tokens": 81, "prompt_tokens": 275, - "total_tokens": 349 + "total_tokens": 356 } } diff --git a/integration-tests/models/__snapshots__/test_flash_gemma3/test_flash_gemma3_image_cow_dog.json b/integration-tests/models/__snapshots__/test_flash_gemma3/test_flash_gemma3_image_cow_dog.json index 0ed2b1e1..6dc6418c 100644 --- a/integration-tests/models/__snapshots__/test_flash_gemma3/test_flash_gemma3_image_cow_dog.json +++ b/integration-tests/models/__snapshots__/test_flash_gemma3/test_flash_gemma3_image_cow_dog.json @@ -1,11 +1,11 @@ { "choices": [ { - "finish_reason": "stop", + "finish_reason": "length", "index": 0, "logprobs": null, "message": { - "content": "That's a fantastic question! However, the image doesn't show a dog. It shows a **Brown Swiss cow** standing on a beach. \n\nBrown Swiss cows are known for their reddish-brown color and distinctive white markings. \n\nIf you'd like, you can send me another image and I’ll do my best to identify it!", + "content": "Based on the image, the animal is a cow, not a dog! It appears to be a **Brazilian Fandangos** cow. \n\nHere's why:\n\n* **Color and markings:** The reddish-brown color and white markings are characteristic of this breed.\n* **Body shape:** They have a distinctive, sturdy build.\n* **Origin:** The Brazilian Fandangos breed originated in Brazil and is known for its ability to thrive in hot, tropical climates", "name": null, "role": "assistant", "tool_calls": null @@ -13,14 +13,14 @@ "usage": null } ], - "created": 1744396697, + "created": 1747123510, "id": "", "model": "google/gemma-3-4b-it", "object": "chat.completion", "system_fingerprint": "3.3.0-dev0-native", "usage": { - "completion_tokens": 75, + "completion_tokens": 100, "prompt_tokens": 279, - "total_tokens": 354 + "total_tokens": 379 } } diff --git a/integration-tests/models/__snapshots__/test_flash_pali_gemma2/test_flash_pali_gemma_image.json b/integration-tests/models/__snapshots__/test_flash_pali_gemma2/test_flash_pali_gemma_image.json index bc75bce4..78f5b5f2 100644 --- a/integration-tests/models/__snapshots__/test_flash_pali_gemma2/test_flash_pali_gemma_image.json +++ b/integration-tests/models/__snapshots__/test_flash_pali_gemma2/test_flash_pali_gemma_image.json @@ -14,43 +14,43 @@ }, { "id": 30234, - "logprob": -2.328125, + "logprob": -2.265625, "special": false, "text": "Brown" }, { "id": 108, - "logprob": -0.12060547, + "logprob": -0.123535156, "special": false, "text": "\n" }, { "id": 3726, - "logprob": -1.7734375, + "logprob": -1.7109375, "special": false, "text": "Car" }, { "id": 108, - "logprob": -0.041503906, + "logprob": -0.036132812, "special": false, "text": "\n" }, { "id": 2915, - "logprob": -1.796875, + "logprob": -1.765625, "special": false, "text": "Color" }, { "id": 108, - "logprob": -0.039794922, + "logprob": -0.044433594, "special": false, "text": "\n" }, { "id": 19178, - "logprob": -1.96875, + "logprob": -2.0, "special": false, "text": "Cool" }, @@ -62,7 +62,7 @@ }, { "id": 40544, - "logprob": -2.1875, + "logprob": -2.125, "special": false, "text": "Decor" }, @@ -72,62 +72,62 @@ "special": false, "text": "\n" }, - { - "id": 13936, - "logprob": -1.6328125, - "special": false, - "text": "Green" - }, { "id": 108, - "logprob": -0.16210938, + "logprob": -1.625, "special": false, "text": "\n" }, - { - "id": 955, - "logprob": -2.015625, - "special": false, - "text": "..." - }, { "id": 108, - "logprob": -0.14746094, + "logprob": -1.921875, "special": false, "text": "\n" }, - { - "id": 955, - "logprob": -0.73828125, - "special": false, - "text": "..." - }, { "id": 108, - "logprob": -0.051513672, + "logprob": -1.5546875, "special": false, "text": "\n" }, - { - "id": 955, - "logprob": -0.34765625, - "special": false, - "text": "..." - }, { "id": 108, - "logprob": -0.020141602, + "logprob": -1.2265625, "special": false, "text": "\n" }, { - "id": 955, - "logprob": -0.11767578, + "id": 108, + "logprob": -0.94921875, "special": false, - "text": "..." + "text": "\n" + }, + { + "id": 108, + "logprob": -1.078125, + "special": false, + "text": "\n" + }, + { + "id": 235336, + "logprob": -1.2265625, + "special": false, + "text": "?" + }, + { + "id": 108, + "logprob": -0.3203125, + "special": false, + "text": "\n" + }, + { + "id": 235336, + "logprob": -0.07861328, + "special": false, + "text": "?" } ], "top_tokens": null }, - "generated_text": "\nBrown\nCar\nColor\nCool\nDecor\nGreen\n...\n...\n...\n..." + "generated_text": "\nBrown\nCar\nColor\nCool\nDecor\n\n\n\n\n\n\n?\n?" } diff --git a/integration-tests/models/test_flash_gemma3.py b/integration-tests/models/test_flash_gemma3.py index 5064f34d..c852f446 100644 --- a/integration-tests/models/test_flash_gemma3.py +++ b/integration-tests/models/test_flash_gemma3.py @@ -53,9 +53,9 @@ async def test_flash_gemma3_image_cow_dog(flash_gemma3, response_snapshot): assert ( response.choices[0].message.content - == "That's a fantastic question! However, the image doesn't show a dog. It shows a **Brown Swiss cow** standing on a beach. \n\nBrown Swiss cows are known for their reddish-brown color and distinctive white markings. \n\nIf you'd like, you can send me another image and I’ll do my best to identify it!" + == "Based on the image, the animal is a cow, not a dog! It appears to be a **Brazilian Fandangos** cow. \n\nHere's why:\n\n* **Color and markings:** The reddish-brown color and white markings are characteristic of this breed.\n* **Body shape:** They have a distinctive, sturdy build.\n* **Origin:** The Brazilian Fandangos breed originated in Brazil and is known for its ability to thrive in hot, tropical climates" ) - assert response.usage["completion_tokens"] == 75 + assert response.usage["completion_tokens"] == 100 assert response == response_snapshot @@ -76,9 +76,9 @@ async def test_flash_gemma3_image_cow(flash_gemma3, response_snapshot): ) assert ( response.choices[0].message.content - == "Here's a description of what's shown in the image:\n\nThe image depicts a brown cow standing on a sandy beach. The beach has turquoise water and a distant island visible in the background. The sky is bright blue with some white clouds. \n\nIt's a quite a humorous and unusual scene – a cow enjoying a day at the beach!" + == "Here's a description of what's shown in the image:\n\nThe image depicts a brown cow standing on a sandy beach. The cow has white markings on its face and ears. Behind the cow is a clear blue ocean with a distant island, and a bright blue sky with some scattered clouds. \n\nIt's a rather charming and unexpected sight – a cow enjoying a beach day!" ) - assert response.usage["completion_tokens"] == 74 + assert response.usage["completion_tokens"] == 81 assert response == response_snapshot diff --git a/integration-tests/models/test_flash_pali_gemma2.py b/integration-tests/models/test_flash_pali_gemma2.py index 23705385..bef9628d 100644 --- a/integration-tests/models/test_flash_pali_gemma2.py +++ b/integration-tests/models/test_flash_pali_gemma2.py @@ -22,8 +22,7 @@ async def test_flash_pali_gemma_image(flash_pali_gemma, response_snapshot): max_new_tokens=20, ) assert ( - response.generated_text - == "\nBrown\nCar\nColor\nCool\nDecor\nGreen\n...\n...\n...\n..." + response.generated_text == "\nBrown\nCar\nColor\nCool\nDecor\n\n\n\n\n\n\n?\n?" ) assert response == response_snapshot