From 308ab7d5b9fe56a174275cae76898458ebb664fb Mon Sep 17 00:00:00 2001
From: Nicolas Patry <patry.nicolas@protonmail.com>
Date: Thu, 17 Aug 2023 06:41:28 +0000
Subject: [PATCH] Update integration test + shard them

---
 .../test_idefics/test_idefics.json            |  66 ++---
 .../test_idefics/test_idefics_load.json       | 260 +++++++++---------
 integration-tests/models/test_idefics.py      |   2 +-
 3 files changed, 164 insertions(+), 164 deletions(-)

diff --git a/integration-tests/models/__snapshots__/test_idefics/test_idefics.json b/integration-tests/models/__snapshots__/test_idefics/test_idefics.json
index 5524bbad..1df5761d 100644
--- a/integration-tests/models/__snapshots__/test_idefics/test_idefics.json
+++ b/integration-tests/models/__snapshots__/test_idefics/test_idefics.json
@@ -11,22 +11,22 @@
       },
       {
         "id": 4911,
-        "logprob": -5.015625,
+        "logprob": -5.3632812,
         "text": "User"
       },
       {
         "id": 29901,
-        "logprob": -0.006000519,
+        "logprob": -0.00762558,
         "text": ":"
       },
       {
         "id": 32000,
-        "logprob": -0.79248047,
+        "logprob": -0.7739258,
         "text": "<fake_token_around_image>"
       },
       {
         "id": 32001,
-        "logprob": -0.0001295805,
+        "logprob": -9.775162e-05,
         "text": "<image>"
       },
       {
@@ -36,67 +36,67 @@
       },
       {
         "id": 1815,
-        "logprob": -4.2734375,
+        "logprob": -4.4140625,
         "text": "Can"
       },
       {
         "id": 366,
-        "logprob": -0.013046265,
+        "logprob": -0.01436615,
         "text": "you"
       },
       {
         "id": 2649,
-        "logprob": -5.0234375,
+        "logprob": -4.9414062,
         "text": "tell"
       },
       {
         "id": 592,
-        "logprob": -0.30688477,
+        "logprob": -0.3005371,
         "text": "me"
       },
       {
         "id": 263,
-        "logprob": -3.5195312,
+        "logprob": -3.5703125,
         "text": "a"
       },
       {
         "id": 1407,
-        "logprob": -9.4375,
+        "logprob": -9.4296875,
         "text": "very"
       },
       {
         "id": 3273,
-        "logprob": -2.0410156,
+        "logprob": -1.9111328,
         "text": "short"
       },
       {
         "id": 5828,
-        "logprob": -0.28393555,
+        "logprob": -0.28881836,
         "text": "story"
       },
       {
         "id": 2729,
-        "logprob": -3.2636719,
+        "logprob": -3.4179688,
         "text": "based"
       },
       {
         "id": 373,
-        "logprob": -0.0007638931,
+        "logprob": -0.00056886673,
         "text": "on"
       },
       {
         "id": 278,
-        "logprob": -0.14746094,
+        "logprob": -0.14123535,
         "text": "the"
       },
       {
         "id": 1967,
-        "logprob": -0.072387695,
+        "logprob": -0.053985596,
         "text": "image"
       },
       {
         "id": 29973,
-        "logprob": -0.1809082,
+        "logprob": -0.15771484,
         "text": "?"
       }
     ],
@@ -104,25 +104,25 @@
     "tokens": [
       {
         "id": 32002,
-        "logprob": -0.007347107,
+        "logprob": -0.004295349,
         "special": true,
         "text": "<end_of_utterance>"
       },
       {
         "id": 29871,
-        "logprob": -6.6161156e-05,
+        "logprob": -7.43866e-05,
         "special": false,
         "text": " "
       },
       {
         "id": 13,
-        "logprob": -2.69413e-05,
+        "logprob": -2.3126602e-05,
         "special": false,
         "text": "\n"
       },
       {
         "id": 7900,
-        "logprob": -3.8146973e-06,
+        "logprob": -3.9339066e-06,
         "special": false,
         "text": "Ass"
       },
@@ -134,35 +134,35 @@
       },
       {
         "id": 29901,
-        "logprob": -2.861023e-06,
+        "logprob": -2.6226044e-06,
         "special": false,
         "text": ":"
       },
       {
         "id": 319,
-        "logprob": -0.90234375,
+        "logprob": -0.87841797,
         "special": false,
         "text": " A"
       },
       {
-        "id": 696,
-        "logprob": -1.4306641,
+        "id": 521,
+        "logprob": -1.3837891,
         "special": false,
-        "text": " ro"
+        "text": " ch"
       },
       {
-        "id": 15664,
-        "logprob": -0.0006227493,
+        "id": 21475,
+        "logprob": -0.00051641464,
         "special": false,
-        "text": "oster"
+        "text": "icken"
       },
       {
-        "id": 15028,
-        "logprob": -1.1425781,
+        "id": 338,
+        "logprob": -1.1435547,
         "special": false,
-        "text": " stands"
+        "text": " is"
       }
     ]
   },
-  "generated_text": "\nAssistant: A rooster stands"
+  "generated_text": "\nAssistant: A chicken is"
 }
diff --git a/integration-tests/models/__snapshots__/test_idefics/test_idefics_load.json b/integration-tests/models/__snapshots__/test_idefics/test_idefics_load.json
index 9ee62354..e0dfe96c 100644
--- a/integration-tests/models/__snapshots__/test_idefics/test_idefics_load.json
+++ b/integration-tests/models/__snapshots__/test_idefics/test_idefics_load.json
@@ -12,22 +12,22 @@
         },
         {
           "id": 4911,
-          "logprob": -5.015625,
+          "logprob": -5.3476562,
           "text": "User"
         },
         {
           "id": 29901,
-          "logprob": -0.006000519,
+          "logprob": -0.0075531006,
           "text": ":"
         },
         {
           "id": 32000,
-          "logprob": -0.79248047,
+          "logprob": -0.7729492,
           "text": "<fake_token_around_image>"
         },
         {
           "id": 32001,
-          "logprob": -0.0001295805,
+          "logprob": -9.787083e-05,
           "text": "<image>"
         },
         {
@@ -37,67 +37,67 @@
         },
         {
           "id": 1815,
-          "logprob": -4.2734375,
+          "logprob": -4.4296875,
           "text": "Can"
         },
         {
           "id": 366,
-          "logprob": -0.013046265,
+          "logprob": -0.01424408,
           "text": "you"
         },
         {
           "id": 2649,
-          "logprob": -5.0234375,
+          "logprob": -4.9335938,
           "text": "tell"
         },
         {
           "id": 592,
-          "logprob": -0.30688477,
+          "logprob": -0.2993164,
           "text": "me"
         },
         {
           "id": 263,
-          "logprob": -3.5195312,
+          "logprob": -3.5664062,
           "text": "a"
         },
         {
           "id": 1407,
-          "logprob": -9.4375,
+          "logprob": -9.4453125,
           "text": "very"
         },
         {
           "id": 3273,
-          "logprob": -2.0410156,
+          "logprob": -1.9306641,
           "text": "short"
         },
         {
           "id": 5828,
-          "logprob": -0.28393555,
+          "logprob": -0.2836914,
           "text": "story"
         },
         {
           "id": 2729,
-          "logprob": -3.2636719,
+          "logprob": -3.4179688,
           "text": "based"
         },
         {
           "id": 373,
-          "logprob": -0.0007638931,
+          "logprob": -0.00056934357,
           "text": "on"
         },
         {
           "id": 278,
-          "logprob": -0.14746094,
+          "logprob": -0.13928223,
           "text": "the"
         },
         {
           "id": 1967,
-          "logprob": -0.072387695,
+          "logprob": -0.05355835,
           "text": "image"
         },
         {
           "id": 29973,
-          "logprob": -0.1809082,
+          "logprob": -0.15771484,
           "text": "?"
         }
       ],
@@ -105,25 +105,25 @@
       "tokens": [
         {
           "id": 32002,
-          "logprob": -0.007347107,
+          "logprob": -0.004333496,
           "special": true,
           "text": "<end_of_utterance>"
         },
         {
           "id": 29871,
-          "logprob": -6.604195e-05,
+          "logprob": -7.426739e-05,
           "special": false,
           "text": " "
         },
         {
           "id": 13,
-          "logprob": -2.69413e-05,
+          "logprob": -2.348423e-05,
           "special": false,
           "text": "\n"
         },
         {
           "id": 7900,
-          "logprob": -3.8146973e-06,
+          "logprob": -3.9339066e-06,
           "special": false,
           "text": "Ass"
         },
@@ -141,31 +141,31 @@
         },
         {
           "id": 319,
-          "logprob": -0.9111328,
+          "logprob": -0.8828125,
           "special": false,
           "text": " A"
         },
         {
-          "id": 696,
-          "logprob": -1.4238281,
+          "id": 521,
+          "logprob": -1.3759766,
           "special": false,
-          "text": " ro"
+          "text": " ch"
         },
         {
-          "id": 15664,
-          "logprob": -0.0006213188,
+          "id": 21475,
+          "logprob": -0.0005083084,
           "special": false,
-          "text": "oster"
+          "text": "icken"
         },
         {
-          "id": 15028,
-          "logprob": -1.1386719,
+          "id": 338,
+          "logprob": -1.1367188,
           "special": false,
-          "text": " stands"
+          "text": " is"
         }
       ]
     },
-    "generated_text": "\nAssistant: A rooster stands"
+    "generated_text": "\nAssistant: A chicken is"
   },
   {
     "details": {
@@ -180,22 +180,22 @@
         },
         {
           "id": 4911,
-          "logprob": -5.0195312,
+          "logprob": -5.3476562,
           "text": "User"
         },
         {
           "id": 29901,
-          "logprob": -0.006023407,
+          "logprob": -0.0075531006,
           "text": ":"
         },
         {
           "id": 32000,
-          "logprob": -0.7915039,
+          "logprob": -0.7729492,
           "text": "<fake_token_around_image>"
         },
         {
           "id": 32001,
-          "logprob": -0.00013053417,
+          "logprob": -9.787083e-05,
           "text": "<image>"
         },
         {
@@ -205,67 +205,67 @@
         },
         {
           "id": 1815,
-          "logprob": -4.2773438,
+          "logprob": -4.4296875,
           "text": "Can"
         },
         {
           "id": 366,
-          "logprob": -0.013046265,
+          "logprob": -0.01423645,
           "text": "you"
         },
         {
           "id": 2649,
-          "logprob": -5.0234375,
+          "logprob": -4.9335938,
           "text": "tell"
         },
         {
           "id": 592,
-          "logprob": -0.3059082,
+          "logprob": -0.2993164,
           "text": "me"
         },
         {
           "id": 263,
-          "logprob": -3.5253906,
+          "logprob": -3.5664062,
           "text": "a"
         },
         {
           "id": 1407,
-          "logprob": -9.4375,
+          "logprob": -9.4453125,
           "text": "very"
         },
         {
           "id": 3273,
-          "logprob": -2.0410156,
+          "logprob": -1.9306641,
           "text": "short"
         },
         {
           "id": 5828,
-          "logprob": -0.28344727,
+          "logprob": -0.2836914,
           "text": "story"
         },
         {
           "id": 2729,
-          "logprob": -3.2617188,
+          "logprob": -3.4179688,
           "text": "based"
         },
         {
           "id": 373,
-          "logprob": -0.00075006485,
+          "logprob": -0.00056934357,
           "text": "on"
         },
         {
           "id": 278,
-          "logprob": -0.14746094,
+          "logprob": -0.13928223,
           "text": "the"
         },
         {
           "id": 1967,
-          "logprob": -0.07305908,
+          "logprob": -0.05355835,
           "text": "image"
         },
         {
           "id": 29973,
-          "logprob": -0.18078613,
+          "logprob": -0.15771484,
           "text": "?"
         }
       ],
@@ -273,25 +273,25 @@
       "tokens": [
         {
           "id": 32002,
-          "logprob": -0.007396698,
+          "logprob": -0.004333496,
           "special": true,
           "text": "<end_of_utterance>"
         },
         {
           "id": 29871,
-          "logprob": -6.663799e-05,
+          "logprob": -7.4505806e-05,
           "special": false,
           "text": " "
         },
         {
           "id": 13,
-          "logprob": -2.670288e-05,
+          "logprob": -2.3722649e-05,
           "special": false,
           "text": "\n"
         },
         {
           "id": 7900,
-          "logprob": -3.8146973e-06,
+          "logprob": -3.9339066e-06,
           "special": false,
           "text": "Ass"
         },
@@ -309,31 +309,31 @@
         },
         {
           "id": 319,
-          "logprob": -0.8979492,
+          "logprob": -0.8828125,
           "special": false,
           "text": " A"
         },
         {
-          "id": 696,
-          "logprob": -1.4296875,
+          "id": 521,
+          "logprob": -1.3759766,
           "special": false,
-          "text": " ro"
+          "text": " ch"
         },
         {
-          "id": 15664,
-          "logprob": -0.0006093979,
+          "id": 21475,
+          "logprob": -0.00050878525,
           "special": false,
-          "text": "oster"
+          "text": "icken"
         },
         {
-          "id": 15028,
-          "logprob": -1.1474609,
+          "id": 338,
+          "logprob": -1.1367188,
           "special": false,
-          "text": " stands"
+          "text": " is"
         }
       ]
     },
-    "generated_text": "\nAssistant: A rooster stands"
+    "generated_text": "\nAssistant: A chicken is"
   },
   {
     "details": {
@@ -348,22 +348,22 @@
         },
         {
           "id": 4911,
-          "logprob": -5.0195312,
+          "logprob": -5.3476562,
           "text": "User"
         },
         {
           "id": 29901,
-          "logprob": -0.006023407,
+          "logprob": -0.0075531006,
           "text": ":"
         },
         {
           "id": 32000,
-          "logprob": -0.7915039,
+          "logprob": -0.7729492,
           "text": "<fake_token_around_image>"
         },
         {
           "id": 32001,
-          "logprob": -0.00013029575,
+          "logprob": -9.775162e-05,
           "text": "<image>"
         },
         {
@@ -373,67 +373,67 @@
         },
         {
           "id": 1815,
-          "logprob": -4.2773438,
+          "logprob": -4.4296875,
           "text": "Can"
         },
         {
           "id": 366,
-          "logprob": -0.013046265,
+          "logprob": -0.01424408,
           "text": "you"
         },
         {
           "id": 2649,
-          "logprob": -5.0234375,
+          "logprob": -4.9335938,
           "text": "tell"
         },
         {
           "id": 592,
-          "logprob": -0.3059082,
+          "logprob": -0.2993164,
           "text": "me"
         },
         {
           "id": 263,
-          "logprob": -3.5253906,
+          "logprob": -3.5664062,
           "text": "a"
         },
         {
           "id": 1407,
-          "logprob": -9.4375,
+          "logprob": -9.4453125,
           "text": "very"
         },
         {
           "id": 3273,
-          "logprob": -2.0410156,
+          "logprob": -1.9306641,
           "text": "short"
         },
         {
           "id": 5828,
-          "logprob": -0.28344727,
+          "logprob": -0.2836914,
           "text": "story"
         },
         {
           "id": 2729,
-          "logprob": -3.2617188,
+          "logprob": -3.4179688,
           "text": "based"
         },
         {
           "id": 373,
-          "logprob": -0.00075006485,
+          "logprob": -0.00056934357,
           "text": "on"
         },
         {
           "id": 278,
-          "logprob": -0.14746094,
+          "logprob": -0.13928223,
           "text": "the"
         },
         {
           "id": 1967,
-          "logprob": -0.07305908,
+          "logprob": -0.05355835,
           "text": "image"
         },
         {
           "id": 29973,
-          "logprob": -0.18078613,
+          "logprob": -0.15771484,
           "text": "?"
         }
       ],
@@ -441,25 +441,25 @@
       "tokens": [
         {
           "id": 32002,
-          "logprob": -0.007396698,
+          "logprob": -0.004333496,
           "special": true,
           "text": "<end_of_utterance>"
         },
         {
           "id": 29871,
-          "logprob": -6.6399574e-05,
+          "logprob": -7.43866e-05,
           "special": false,
           "text": " "
         },
         {
           "id": 13,
-          "logprob": -2.682209e-05,
+          "logprob": -2.360344e-05,
           "special": false,
           "text": "\n"
         },
         {
           "id": 7900,
-          "logprob": -3.8146973e-06,
+          "logprob": -3.9339066e-06,
           "special": false,
           "text": "Ass"
         },
@@ -471,37 +471,37 @@
         },
         {
           "id": 29901,
-          "logprob": -2.861023e-06,
+          "logprob": -2.7418137e-06,
           "special": false,
           "text": ":"
         },
         {
           "id": 319,
-          "logprob": -0.8979492,
+          "logprob": -0.8828125,
           "special": false,
           "text": " A"
         },
         {
-          "id": 696,
-          "logprob": -1.4296875,
+          "id": 521,
+          "logprob": -1.3759766,
           "special": false,
-          "text": " ro"
+          "text": " ch"
         },
         {
-          "id": 15664,
-          "logprob": -0.0006093979,
+          "id": 21475,
+          "logprob": -0.00050878525,
           "special": false,
-          "text": "oster"
+          "text": "icken"
         },
         {
-          "id": 15028,
-          "logprob": -1.1474609,
+          "id": 338,
+          "logprob": -1.1367188,
           "special": false,
-          "text": " stands"
+          "text": " is"
         }
       ]
     },
-    "generated_text": "\nAssistant: A rooster stands"
+    "generated_text": "\nAssistant: A chicken is"
   },
   {
     "details": {
@@ -516,22 +516,22 @@
         },
         {
           "id": 4911,
-          "logprob": -5.0195312,
+          "logprob": -5.3632812,
           "text": "User"
         },
         {
           "id": 29901,
-          "logprob": -0.006023407,
+          "logprob": -0.00762558,
           "text": ":"
         },
         {
           "id": 32000,
-          "logprob": -0.7915039,
+          "logprob": -0.7739258,
           "text": "<fake_token_around_image>"
         },
         {
           "id": 32001,
-          "logprob": -0.00013041496,
+          "logprob": -9.775162e-05,
           "text": "<image>"
         },
         {
@@ -541,67 +541,67 @@
         },
         {
           "id": 1815,
-          "logprob": -4.2773438,
+          "logprob": -4.4140625,
           "text": "Can"
         },
         {
           "id": 366,
-          "logprob": -0.013046265,
+          "logprob": -0.01436615,
           "text": "you"
         },
         {
           "id": 2649,
-          "logprob": -5.0234375,
+          "logprob": -4.9414062,
           "text": "tell"
         },
         {
           "id": 592,
-          "logprob": -0.3059082,
+          "logprob": -0.3005371,
           "text": "me"
         },
         {
           "id": 263,
-          "logprob": -3.5253906,
+          "logprob": -3.5703125,
           "text": "a"
         },
         {
           "id": 1407,
-          "logprob": -9.4375,
+          "logprob": -9.4296875,
           "text": "very"
         },
         {
           "id": 3273,
-          "logprob": -2.0410156,
+          "logprob": -1.9111328,
           "text": "short"
         },
         {
           "id": 5828,
-          "logprob": -0.28344727,
+          "logprob": -0.28881836,
           "text": "story"
         },
         {
           "id": 2729,
-          "logprob": -3.2617188,
+          "logprob": -3.4179688,
           "text": "based"
         },
         {
           "id": 373,
-          "logprob": -0.00075006485,
+          "logprob": -0.00056886673,
           "text": "on"
         },
         {
           "id": 278,
-          "logprob": -0.14746094,
+          "logprob": -0.14123535,
           "text": "the"
         },
         {
           "id": 1967,
-          "logprob": -0.07305908,
+          "logprob": -0.053985596,
           "text": "image"
         },
         {
           "id": 29973,
-          "logprob": -0.18078613,
+          "logprob": -0.15771484,
           "text": "?"
         }
       ],
@@ -609,25 +609,25 @@
       "tokens": [
         {
           "id": 32002,
-          "logprob": -0.007396698,
+          "logprob": -0.004295349,
           "special": true,
           "text": "<end_of_utterance>"
         },
         {
           "id": 29871,
-          "logprob": -6.6399574e-05,
+          "logprob": -7.43866e-05,
           "special": false,
           "text": " "
         },
         {
           "id": 13,
-          "logprob": -2.69413e-05,
+          "logprob": -2.3126602e-05,
           "special": false,
           "text": "\n"
         },
         {
           "id": 7900,
-          "logprob": -3.8146973e-06,
+          "logprob": -3.9339066e-06,
           "special": false,
           "text": "Ass"
         },
@@ -639,36 +639,36 @@
         },
         {
           "id": 29901,
-          "logprob": -2.861023e-06,
+          "logprob": -2.6226044e-06,
           "special": false,
           "text": ":"
         },
         {
           "id": 319,
-          "logprob": -0.8979492,
+          "logprob": -0.87841797,
           "special": false,
           "text": " A"
         },
         {
-          "id": 696,
-          "logprob": -1.4296875,
+          "id": 521,
+          "logprob": -1.3837891,
           "special": false,
-          "text": " ro"
+          "text": " ch"
         },
         {
-          "id": 15664,
-          "logprob": -0.0006093979,
+          "id": 21475,
+          "logprob": -0.00051641464,
           "special": false,
-          "text": "oster"
+          "text": "icken"
         },
         {
-          "id": 15028,
-          "logprob": -1.1474609,
+          "id": 338,
+          "logprob": -1.1435547,
           "special": false,
-          "text": " stands"
+          "text": " is"
         }
       ]
     },
-    "generated_text": "\nAssistant: A rooster stands"
+    "generated_text": "\nAssistant: A chicken is"
   }
 ]
diff --git a/integration-tests/models/test_idefics.py b/integration-tests/models/test_idefics.py
index 6c2afe24..5659dd5c 100644
--- a/integration-tests/models/test_idefics.py
+++ b/integration-tests/models/test_idefics.py
@@ -4,7 +4,7 @@ import pytest
 @pytest.fixture(scope="module")
 def idefics_handle(launcher):
     with launcher(
-        "HuggingFaceM4/idefics-9b-instruct", num_shard=1
+        "HuggingFaceM4/idefics-9b-instruct", num_shard=2
     ) as handle:
         yield handle