diff --git a/integration-tests/models/__snapshots__/test_flash_awq/test_flash_llama_awq_all_params.json b/integration-tests/models/__snapshots__/test_flash_awq/test_flash_llama_awq_all_params.json
index edb588ea..d16d34f9 100644
--- a/integration-tests/models/__snapshots__/test_flash_awq/test_flash_llama_awq_all_params.json
+++ b/integration-tests/models/__snapshots__/test_flash_awq/test_flash_llama_awq_all_params.json
@@ -39,60 +39,61 @@
         "text": "\n"
       },
       {
-        "id": 1576,
-        "logprob": -1.2138672,
+        "id": 4013,
+        "logprob": -2.203125,
         "special": false,
-        "text": "The"
+        "text": "This"
       },
       {
-        "id": 3611,
-        "logprob": -0.9482422,
+        "id": 1139,
+        "logprob": -0.23693848,
         "special": false,
-        "text": " title"
+        "text": " question"
       },
       {
-        "id": 310,
+        "id": 756,
         "logprob": 0.0,
         "special": false,
-        "text": " of"
+        "text": " has"
       },
       {
-        "id": 445,
-        "logprob": -0.140625,
+        "id": 1063,
+        "logprob": -0.076538086,
         "special": false,
-        "text": " this"
+        "text": " been"
       },
       {
-        "id": 1400,
-        "logprob": -0.2286377,
-        "special": false,
-        "text": " post"
-      },
-      {
-        "id": 1795,
-        "logprob": -0.5253906,
-        "special": false,
-        "text": " might"
-      },
-      {
-        "id": 505,
-        "logprob": -0.30444336,
-        "special": false,
-        "text": " have"
-      },
-      {
-        "id": 366,
+        "id": 4433,
         "logprob": 0.0,
         "special": false,
-        "text": " you"
+        "text": " asked"
       },
       {
-        "id": 7291,
-        "logprob": -0.68652344,
+        "id": 1784,
+        "logprob": -1.1367188,
         "special": false,
-        "text": " thinking"
+        "text": " many"
+      },
+      {
+        "id": 3064,
+        "logprob": 0.0,
+        "special": false,
+        "text": " times"
+      },
+      {
+        "id": 322,
+        "logprob": -1.7460938,
+        "special": false,
+        "text": " and"
+      },
+      {
+        "id": 306,
+        "logprob": 0.0,
+        "special": false,
+        "text": " I"
       }
-    ]
+    ],
+    "top_tokens": null
   },
-  "generated_text": "What is Deep Learning?\nThe title of this post might have you thinking"
+  "generated_text": "What is Deep Learning?\nThis question has been asked many times and I"
 }
diff --git a/integration-tests/models/__snapshots__/test_flash_falcon/test_flash_falcon_all_params.json b/integration-tests/models/__snapshots__/test_flash_falcon/test_flash_falcon_all_params.json
index c8f2920e..cd35186d 100644
--- a/integration-tests/models/__snapshots__/test_flash_falcon/test_flash_falcon_all_params.json
+++ b/integration-tests/models/__snapshots__/test_flash_falcon/test_flash_falcon_all_params.json
@@ -11,22 +11,22 @@
       },
       {
         "id": 1622,
-        "logprob": -7.7421875,
+        "logprob": -7.8125,
         "text": "af"
       },
       {
         "id": 249,
-        "logprob": -4.484375,
+        "logprob": -4.5,
         "text": "at"
       },
       {
         "id": 1480,
-        "logprob": -10.890625,
+        "logprob": -10.875,
         "text": "ron"
       },
       {
         "id": 37,
-        "logprob": -3.6757812,
+        "logprob": -3.6875,
         "text": ":"
       }
     ],
@@ -34,65 +34,65 @@
     "tokens": [
       {
         "id": 836,
-        "logprob": -1.2822266,
+        "logprob": -1.265625,
         "special": false,
         "text": " i"
       },
       {
         "id": 18,
-        "logprob": -0.14953613,
+        "logprob": -0.119628906,
         "special": false,
         "text": "'"
       },
       {
-        "id": 88,
-        "logprob": -0.11138916,
+        "id": 298,
+        "logprob": -2.265625,
         "special": false,
-        "text": "m"
+        "text": "ve"
       },
       {
-        "id": 416,
-        "logprob": -0.074401855,
+        "id": 650,
+        "logprob": -0.49804688,
         "special": false,
-        "text": " not"
+        "text": " been"
       },
       {
-        "id": 830,
-        "logprob": -2.2148438,
+        "id": 1241,
+        "logprob": 0.0,
         "special": false,
-        "text": " even"
-      },
-      {
-        "id": 1310,
-        "logprob": -0.19958496,
-        "special": false,
-        "text": " sure"
-      },
-      {
-        "id": 637,
-        "logprob": -0.19787598,
-        "special": false,
-        "text": " what"
+        "text": " using"
       },
       {
         "id": 334,
-        "logprob": -0.6386719,
+        "logprob": 0.0,
         "special": false,
         "text": " it"
       },
       {
-        "id": 304,
-        "logprob": 0.0,
+        "id": 312,
+        "logprob": -1.2421875,
         "special": false,
-        "text": " is"
+        "text": " for"
+      },
+      {
+        "id": 909,
+        "logprob": -0.99609375,
+        "special": false,
+        "text": " years"
       },
       {
         "id": 193,
-        "logprob": 0.0,
+        "logprob": -0.30273438,
         "special": false,
         "text": "\n"
+      },
+      {
+        "id": 807,
+        "logprob": -1.078125,
+        "special": false,
+        "text": "ik"
       }
     ]
   },
-  "generated_text": "Girafatron is obsessed with giraffes, the most glorious animal on the face of this Earth. Giraftron believes all other animals are irrelevant when compared to the glorious majesty of the giraffe.\nDaniel: Hello, Girafatron!\nGirafatron: i'm not even sure what it is\n"
+  "generated_text": "Girafatron is obsessed with giraffes, the most glorious animal on the face of this Earth. Giraftron believes all other animals are irrelevant when compared to the glorious majesty of the giraffe.\nDaniel: Hello, Girafatron!\nGirafatron: i've been using it for years\nik"
 }
diff --git a/integration-tests/models/__snapshots__/test_flash_falcon/test_flash_falcon_load.json b/integration-tests/models/__snapshots__/test_flash_falcon/test_flash_falcon_load.json
index 84b16f0b..90a35eb7 100644
--- a/integration-tests/models/__snapshots__/test_flash_falcon/test_flash_falcon_load.json
+++ b/integration-tests/models/__snapshots__/test_flash_falcon/test_flash_falcon_load.json
@@ -6,96 +6,376 @@
       "generated_tokens": 10,
       "prefill": [
         {
-          "id": 1562,
+          "id": 50,
           "logprob": null,
-          "text": "What"
+          "text": "G"
+        },
+        {
+          "id": 330,
+          "logprob": -5.96875,
+          "text": "ir"
+        },
+        {
+          "id": 1622,
+          "logprob": -5.6171875,
+          "text": "af"
+        },
+        {
+          "id": 249,
+          "logprob": -6.5039062,
+          "text": "at"
+        },
+        {
+          "id": 1480,
+          "logprob": -8.0703125,
+          "text": "ron"
         },
         {
           "id": 304,
-          "logprob": -1.5371094,
+          "logprob": -2.328125,
           "text": " is"
         },
         {
-          "id": 12604,
-          "logprob": -8.7109375,
-          "text": " Deep"
+          "id": 23866,
+          "logprob": -9.59375,
+          "text": " obsessed"
         },
         {
-          "id": 9078,
-          "logprob": -1.9384766,
-          "text": " Learning"
+          "id": 335,
+          "logprob": -0.04837036,
+          "text": " with"
         },
         {
-          "id": 42,
-          "logprob": -0.32958984,
-          "text": "?"
+          "id": 26680,
+          "logprob": -3.9960938,
+          "text": " gir"
+        },
+        {
+          "id": 1903,
+          "logprob": -0.07525635,
+          "text": "aff"
+        },
+        {
+          "id": 255,
+          "logprob": -0.006790161,
+          "text": "es"
+        },
+        {
+          "id": 23,
+          "logprob": -1.546875,
+          "text": ","
+        },
+        {
+          "id": 248,
+          "logprob": -4.3320312,
+          "text": " the"
+        },
+        {
+          "id": 758,
+          "logprob": -3.7363281,
+          "text": " most"
+        },
+        {
+          "id": 21735,
+          "logprob": -5.109375,
+          "text": " glorious"
+        },
+        {
+          "id": 5985,
+          "logprob": -2.09375,
+          "text": " animal"
+        },
+        {
+          "id": 313,
+          "logprob": -1.1845703,
+          "text": " on"
+        },
+        {
+          "id": 248,
+          "logprob": -0.77734375,
+          "text": " the"
+        },
+        {
+          "id": 1936,
+          "logprob": -2.3828125,
+          "text": " face"
+        },
+        {
+          "id": 275,
+          "logprob": -0.0044403076,
+          "text": " of"
+        },
+        {
+          "id": 414,
+          "logprob": -1.9667969,
+          "text": " this"
+        },
+        {
+          "id": 6490,
+          "logprob": -2.0449219,
+          "text": " Earth"
+        },
+        {
+          "id": 25,
+          "logprob": -0.28198242,
+          "text": "."
+        },
+        {
+          "id": 401,
+          "logprob": -7.921875,
+          "text": " G"
+        },
+        {
+          "id": 6013,
+          "logprob": -2.2714844,
+          "text": "ira"
+        },
+        {
+          "id": 694,
+          "logprob": -0.62353516,
+          "text": "ft"
+        },
+        {
+          "id": 1480,
+          "logprob": -0.20947266,
+          "text": "ron"
+        },
+        {
+          "id": 9369,
+          "logprob": -4.5507812,
+          "text": " believes"
+        },
+        {
+          "id": 455,
+          "logprob": -4.5625,
+          "text": " all"
+        },
+        {
+          "id": 599,
+          "logprob": -2.7402344,
+          "text": " other"
+        },
+        {
+          "id": 5632,
+          "logprob": -0.21899414,
+          "text": " animals"
+        },
+        {
+          "id": 362,
+          "logprob": -0.76708984,
+          "text": " are"
+        },
+        {
+          "id": 23981,
+          "logprob": -4.9960938,
+          "text": " irrelevant"
+        },
+        {
+          "id": 635,
+          "logprob": -4.234375,
+          "text": " when"
+        },
+        {
+          "id": 4354,
+          "logprob": -0.5131836,
+          "text": " compared"
+        },
+        {
+          "id": 271,
+          "logprob": -0.103515625,
+          "text": " to"
+        },
+        {
+          "id": 248,
+          "logprob": -0.58447266,
+          "text": " the"
+        },
+        {
+          "id": 21735,
+          "logprob": -3.6796875,
+          "text": " glorious"
+        },
+        {
+          "id": 64398,
+          "logprob": -1.8222656,
+          "text": " majesty"
+        },
+        {
+          "id": 275,
+          "logprob": -0.23583984,
+          "text": " of"
+        },
+        {
+          "id": 248,
+          "logprob": -0.3544922,
+          "text": " the"
+        },
+        {
+          "id": 26680,
+          "logprob": -0.24609375,
+          "text": " gir"
+        },
+        {
+          "id": 23226,
+          "logprob": -0.02960205,
+          "text": "affe"
+        },
+        {
+          "id": 25,
+          "logprob": -0.17358398,
+          "text": "."
+        },
+        {
+          "id": 193,
+          "logprob": -1.3925781,
+          "text": "\n"
+        },
+        {
+          "id": 23626,
+          "logprob": -10.0625,
+          "text": "Daniel"
+        },
+        {
+          "id": 37,
+          "logprob": -4.5898438,
+          "text": ":"
+        },
+        {
+          "id": 23090,
+          "logprob": -6.9375,
+          "text": " Hello"
+        },
+        {
+          "id": 23,
+          "logprob": -0.99365234,
+          "text": ","
+        },
+        {
+          "id": 29033,
+          "logprob": -2.2304688,
+          "text": " Gir"
+        },
+        {
+          "id": 1622,
+          "logprob": -0.107788086,
+          "text": "af"
+        },
+        {
+          "id": 249,
+          "logprob": -0.04257202,
+          "text": "at"
+        },
+        {
+          "id": 1480,
+          "logprob": -0.0024871826,
+          "text": "ron"
+        },
+        {
+          "id": 12,
+          "logprob": -1.4277344,
+          "text": "!"
+        },
+        {
+          "id": 193,
+          "logprob": -1.1005859,
+          "text": "\n"
+        },
+        {
+          "id": 50,
+          "logprob": -0.056915283,
+          "text": "G"
+        },
+        {
+          "id": 330,
+          "logprob": -0.1315918,
+          "text": "ir"
+        },
+        {
+          "id": 1622,
+          "logprob": -0.0071105957,
+          "text": "af"
+        },
+        {
+          "id": 249,
+          "logprob": -0.008453369,
+          "text": "at"
+        },
+        {
+          "id": 1480,
+          "logprob": -0.0006928444,
+          "text": "ron"
+        },
+        {
+          "id": 37,
+          "logprob": -0.0074920654,
+          "text": ":"
         }
       ],
       "seed": null,
       "tokens": [
+        {
+          "id": 23090,
+          "logprob": -1.828125,
+          "special": false,
+          "text": " Hello"
+        },
+        {
+          "id": 23,
+          "logprob": -0.3178711,
+          "special": false,
+          "text": ","
+        },
+        {
+          "id": 8156,
+          "logprob": -0.23925781,
+          "special": false,
+          "text": " Daniel"
+        },
+        {
+          "id": 12,
+          "logprob": -0.5698242,
+          "special": false,
+          "text": "!"
+        },
         {
           "id": 193,
-          "logprob": -0.27563477,
+          "logprob": -0.61279297,
           "special": false,
           "text": "\n"
         },
         {
-          "id": 27319,
-          "logprob": -1.1015625,
+          "id": 23626,
+          "logprob": -0.4177246,
           "special": false,
-          "text": "Deep"
+          "text": "Daniel"
         },
         {
-          "id": 2860,
-          "logprob": -0.5292969,
+          "id": 37,
+          "logprob": -0.0023345947,
           "special": false,
-          "text": " learning"
+          "text": ":"
         },
         {
-          "id": 304,
-          "logprob": -0.26538086,
+          "id": 1634,
+          "logprob": -2.0605469,
           "special": false,
-          "text": " is"
+          "text": " What"
         },
         {
-          "id": 241,
-          "logprob": -0.41430664,
+          "id": 18,
+          "logprob": -1.5283203,
           "special": false,
-          "text": " a"
+          "text": "'"
         },
         {
-          "id": 30095,
-          "logprob": -1.2724609,
+          "id": 94,
+          "logprob": -0.007965088,
           "special": false,
-          "text": " subset"
-        },
-        {
-          "id": 275,
-          "logprob": -0.0115737915,
-          "special": false,
-          "text": " of"
-        },
-        {
-          "id": 3414,
-          "logprob": -0.2409668,
-          "special": false,
-          "text": " machine"
-        },
-        {
-          "id": 2860,
-          "logprob": -0.008323669,
-          "special": false,
-          "text": " learning"
-        },
-        {
-          "id": 325,
-          "logprob": -1.1533203,
-          "special": false,
-          "text": " that"
+          "text": "s"
         }
       ]
     },
-    "generated_text": "\nDeep learning is a subset of machine learning that"
+    "generated_text": " Hello, Daniel!\nDaniel: What's"
   },
   {
     "details": {
@@ -104,96 +384,376 @@
       "generated_tokens": 10,
       "prefill": [
         {
-          "id": 1562,
+          "id": 50,
           "logprob": null,
-          "text": "What"
+          "text": "G"
+        },
+        {
+          "id": 330,
+          "logprob": -5.96875,
+          "text": "ir"
+        },
+        {
+          "id": 1622,
+          "logprob": -5.6171875,
+          "text": "af"
+        },
+        {
+          "id": 249,
+          "logprob": -6.5,
+          "text": "at"
+        },
+        {
+          "id": 1480,
+          "logprob": -8.0703125,
+          "text": "ron"
         },
         {
           "id": 304,
-          "logprob": -1.5371094,
+          "logprob": -2.328125,
           "text": " is"
         },
         {
-          "id": 12604,
-          "logprob": -8.7109375,
-          "text": " Deep"
+          "id": 23866,
+          "logprob": -9.59375,
+          "text": " obsessed"
         },
         {
-          "id": 9078,
-          "logprob": -1.9384766,
-          "text": " Learning"
+          "id": 335,
+          "logprob": -0.048339844,
+          "text": " with"
         },
         {
-          "id": 42,
-          "logprob": -0.32958984,
-          "text": "?"
+          "id": 26680,
+          "logprob": -4.0,
+          "text": " gir"
+        },
+        {
+          "id": 1903,
+          "logprob": -0.07531738,
+          "text": "aff"
+        },
+        {
+          "id": 255,
+          "logprob": -0.006793976,
+          "text": "es"
+        },
+        {
+          "id": 23,
+          "logprob": -1.5478516,
+          "text": ","
+        },
+        {
+          "id": 248,
+          "logprob": -4.3320312,
+          "text": " the"
+        },
+        {
+          "id": 758,
+          "logprob": -3.7363281,
+          "text": " most"
+        },
+        {
+          "id": 21735,
+          "logprob": -5.1132812,
+          "text": " glorious"
+        },
+        {
+          "id": 5985,
+          "logprob": -2.0957031,
+          "text": " animal"
+        },
+        {
+          "id": 313,
+          "logprob": -1.1835938,
+          "text": " on"
+        },
+        {
+          "id": 248,
+          "logprob": -0.77685547,
+          "text": " the"
+        },
+        {
+          "id": 1936,
+          "logprob": -2.3808594,
+          "text": " face"
+        },
+        {
+          "id": 275,
+          "logprob": -0.004436493,
+          "text": " of"
+        },
+        {
+          "id": 414,
+          "logprob": -1.9638672,
+          "text": " this"
+        },
+        {
+          "id": 6490,
+          "logprob": -2.0449219,
+          "text": " Earth"
+        },
+        {
+          "id": 25,
+          "logprob": -0.28198242,
+          "text": "."
+        },
+        {
+          "id": 401,
+          "logprob": -7.9179688,
+          "text": " G"
+        },
+        {
+          "id": 6013,
+          "logprob": -2.2734375,
+          "text": "ira"
+        },
+        {
+          "id": 694,
+          "logprob": -0.6230469,
+          "text": "ft"
+        },
+        {
+          "id": 1480,
+          "logprob": -0.20947266,
+          "text": "ron"
+        },
+        {
+          "id": 9369,
+          "logprob": -4.5546875,
+          "text": " believes"
+        },
+        {
+          "id": 455,
+          "logprob": -4.5703125,
+          "text": " all"
+        },
+        {
+          "id": 599,
+          "logprob": -2.7382812,
+          "text": " other"
+        },
+        {
+          "id": 5632,
+          "logprob": -0.21948242,
+          "text": " animals"
+        },
+        {
+          "id": 362,
+          "logprob": -0.7661133,
+          "text": " are"
+        },
+        {
+          "id": 23981,
+          "logprob": -4.9960938,
+          "text": " irrelevant"
+        },
+        {
+          "id": 635,
+          "logprob": -4.234375,
+          "text": " when"
+        },
+        {
+          "id": 4354,
+          "logprob": -0.5131836,
+          "text": " compared"
+        },
+        {
+          "id": 271,
+          "logprob": -0.10357666,
+          "text": " to"
+        },
+        {
+          "id": 248,
+          "logprob": -0.58447266,
+          "text": " the"
+        },
+        {
+          "id": 21735,
+          "logprob": -3.6816406,
+          "text": " glorious"
+        },
+        {
+          "id": 64398,
+          "logprob": -1.8203125,
+          "text": " majesty"
+        },
+        {
+          "id": 275,
+          "logprob": -0.23583984,
+          "text": " of"
+        },
+        {
+          "id": 248,
+          "logprob": -0.35473633,
+          "text": " the"
+        },
+        {
+          "id": 26680,
+          "logprob": -0.24572754,
+          "text": " gir"
+        },
+        {
+          "id": 23226,
+          "logprob": -0.029586792,
+          "text": "affe"
+        },
+        {
+          "id": 25,
+          "logprob": -0.17346191,
+          "text": "."
+        },
+        {
+          "id": 193,
+          "logprob": -1.3945312,
+          "text": "\n"
+        },
+        {
+          "id": 23626,
+          "logprob": -10.0625,
+          "text": "Daniel"
+        },
+        {
+          "id": 37,
+          "logprob": -4.59375,
+          "text": ":"
+        },
+        {
+          "id": 23090,
+          "logprob": -6.9375,
+          "text": " Hello"
+        },
+        {
+          "id": 23,
+          "logprob": -0.99316406,
+          "text": ","
+        },
+        {
+          "id": 29033,
+          "logprob": -2.2324219,
+          "text": " Gir"
+        },
+        {
+          "id": 1622,
+          "logprob": -0.10797119,
+          "text": "af"
+        },
+        {
+          "id": 249,
+          "logprob": -0.04248047,
+          "text": "at"
+        },
+        {
+          "id": 1480,
+          "logprob": -0.0024814606,
+          "text": "ron"
+        },
+        {
+          "id": 12,
+          "logprob": -1.4277344,
+          "text": "!"
+        },
+        {
+          "id": 193,
+          "logprob": -1.1005859,
+          "text": "\n"
+        },
+        {
+          "id": 50,
+          "logprob": -0.056884766,
+          "text": "G"
+        },
+        {
+          "id": 330,
+          "logprob": -0.1315918,
+          "text": "ir"
+        },
+        {
+          "id": 1622,
+          "logprob": -0.007095337,
+          "text": "af"
+        },
+        {
+          "id": 249,
+          "logprob": -0.00844574,
+          "text": "at"
+        },
+        {
+          "id": 1480,
+          "logprob": -0.00068998337,
+          "text": "ron"
+        },
+        {
+          "id": 37,
+          "logprob": -0.0074768066,
+          "text": ":"
         }
       ],
       "seed": null,
       "tokens": [
+        {
+          "id": 23090,
+          "logprob": -1.8251953,
+          "special": false,
+          "text": " Hello"
+        },
+        {
+          "id": 23,
+          "logprob": -0.31762695,
+          "special": false,
+          "text": ","
+        },
+        {
+          "id": 8156,
+          "logprob": -0.2388916,
+          "special": false,
+          "text": " Daniel"
+        },
+        {
+          "id": 12,
+          "logprob": -0.5698242,
+          "special": false,
+          "text": "!"
+        },
         {
           "id": 193,
-          "logprob": -0.27563477,
+          "logprob": -0.6152344,
           "special": false,
           "text": "\n"
         },
         {
-          "id": 27319,
-          "logprob": -1.1015625,
+          "id": 23626,
+          "logprob": -0.42211914,
           "special": false,
-          "text": "Deep"
+          "text": "Daniel"
         },
         {
-          "id": 2860,
-          "logprob": -0.5292969,
+          "id": 37,
+          "logprob": -0.002336502,
           "special": false,
-          "text": " learning"
+          "text": ":"
         },
         {
-          "id": 304,
-          "logprob": -0.26538086,
+          "id": 1634,
+          "logprob": -2.0605469,
           "special": false,
-          "text": " is"
+          "text": " What"
         },
         {
-          "id": 241,
-          "logprob": -0.41430664,
+          "id": 18,
+          "logprob": -1.5292969,
           "special": false,
-          "text": " a"
+          "text": "'"
         },
         {
-          "id": 30095,
-          "logprob": -1.2724609,
+          "id": 94,
+          "logprob": -0.007926941,
           "special": false,
-          "text": " subset"
-        },
-        {
-          "id": 275,
-          "logprob": -0.0115737915,
-          "special": false,
-          "text": " of"
-        },
-        {
-          "id": 3414,
-          "logprob": -0.2409668,
-          "special": false,
-          "text": " machine"
-        },
-        {
-          "id": 2860,
-          "logprob": -0.008323669,
-          "special": false,
-          "text": " learning"
-        },
-        {
-          "id": 325,
-          "logprob": -1.1533203,
-          "special": false,
-          "text": " that"
+          "text": "s"
         }
       ]
     },
-    "generated_text": "\nDeep learning is a subset of machine learning that"
+    "generated_text": " Hello, Daniel!\nDaniel: What's"
   },
   {
     "details": {
@@ -202,96 +762,376 @@
       "generated_tokens": 10,
       "prefill": [
         {
-          "id": 1562,
+          "id": 50,
           "logprob": null,
-          "text": "What"
+          "text": "G"
+        },
+        {
+          "id": 330,
+          "logprob": -5.96875,
+          "text": "ir"
+        },
+        {
+          "id": 1622,
+          "logprob": -5.6171875,
+          "text": "af"
+        },
+        {
+          "id": 249,
+          "logprob": -6.5,
+          "text": "at"
+        },
+        {
+          "id": 1480,
+          "logprob": -8.0703125,
+          "text": "ron"
         },
         {
           "id": 304,
-          "logprob": -1.5371094,
+          "logprob": -2.328125,
           "text": " is"
         },
         {
-          "id": 12604,
-          "logprob": -8.7109375,
-          "text": " Deep"
+          "id": 23866,
+          "logprob": -9.59375,
+          "text": " obsessed"
         },
         {
-          "id": 9078,
-          "logprob": -1.9384766,
-          "text": " Learning"
+          "id": 335,
+          "logprob": -0.048339844,
+          "text": " with"
         },
         {
-          "id": 42,
-          "logprob": -0.32958984,
-          "text": "?"
+          "id": 26680,
+          "logprob": -4.0,
+          "text": " gir"
+        },
+        {
+          "id": 1903,
+          "logprob": -0.07531738,
+          "text": "aff"
+        },
+        {
+          "id": 255,
+          "logprob": -0.006793976,
+          "text": "es"
+        },
+        {
+          "id": 23,
+          "logprob": -1.5478516,
+          "text": ","
+        },
+        {
+          "id": 248,
+          "logprob": -4.3320312,
+          "text": " the"
+        },
+        {
+          "id": 758,
+          "logprob": -3.7363281,
+          "text": " most"
+        },
+        {
+          "id": 21735,
+          "logprob": -5.1132812,
+          "text": " glorious"
+        },
+        {
+          "id": 5985,
+          "logprob": -2.0957031,
+          "text": " animal"
+        },
+        {
+          "id": 313,
+          "logprob": -1.1835938,
+          "text": " on"
+        },
+        {
+          "id": 248,
+          "logprob": -0.77685547,
+          "text": " the"
+        },
+        {
+          "id": 1936,
+          "logprob": -2.3808594,
+          "text": " face"
+        },
+        {
+          "id": 275,
+          "logprob": -0.004436493,
+          "text": " of"
+        },
+        {
+          "id": 414,
+          "logprob": -1.9638672,
+          "text": " this"
+        },
+        {
+          "id": 6490,
+          "logprob": -2.0449219,
+          "text": " Earth"
+        },
+        {
+          "id": 25,
+          "logprob": -0.28198242,
+          "text": "."
+        },
+        {
+          "id": 401,
+          "logprob": -7.9179688,
+          "text": " G"
+        },
+        {
+          "id": 6013,
+          "logprob": -2.2734375,
+          "text": "ira"
+        },
+        {
+          "id": 694,
+          "logprob": -0.6230469,
+          "text": "ft"
+        },
+        {
+          "id": 1480,
+          "logprob": -0.20947266,
+          "text": "ron"
+        },
+        {
+          "id": 9369,
+          "logprob": -4.5546875,
+          "text": " believes"
+        },
+        {
+          "id": 455,
+          "logprob": -4.5703125,
+          "text": " all"
+        },
+        {
+          "id": 599,
+          "logprob": -2.7382812,
+          "text": " other"
+        },
+        {
+          "id": 5632,
+          "logprob": -0.21948242,
+          "text": " animals"
+        },
+        {
+          "id": 362,
+          "logprob": -0.7661133,
+          "text": " are"
+        },
+        {
+          "id": 23981,
+          "logprob": -4.9960938,
+          "text": " irrelevant"
+        },
+        {
+          "id": 635,
+          "logprob": -4.234375,
+          "text": " when"
+        },
+        {
+          "id": 4354,
+          "logprob": -0.5131836,
+          "text": " compared"
+        },
+        {
+          "id": 271,
+          "logprob": -0.10357666,
+          "text": " to"
+        },
+        {
+          "id": 248,
+          "logprob": -0.58447266,
+          "text": " the"
+        },
+        {
+          "id": 21735,
+          "logprob": -3.6816406,
+          "text": " glorious"
+        },
+        {
+          "id": 64398,
+          "logprob": -1.8203125,
+          "text": " majesty"
+        },
+        {
+          "id": 275,
+          "logprob": -0.23583984,
+          "text": " of"
+        },
+        {
+          "id": 248,
+          "logprob": -0.35473633,
+          "text": " the"
+        },
+        {
+          "id": 26680,
+          "logprob": -0.24572754,
+          "text": " gir"
+        },
+        {
+          "id": 23226,
+          "logprob": -0.029586792,
+          "text": "affe"
+        },
+        {
+          "id": 25,
+          "logprob": -0.17346191,
+          "text": "."
+        },
+        {
+          "id": 193,
+          "logprob": -1.3945312,
+          "text": "\n"
+        },
+        {
+          "id": 23626,
+          "logprob": -10.0625,
+          "text": "Daniel"
+        },
+        {
+          "id": 37,
+          "logprob": -4.59375,
+          "text": ":"
+        },
+        {
+          "id": 23090,
+          "logprob": -6.9375,
+          "text": " Hello"
+        },
+        {
+          "id": 23,
+          "logprob": -0.99316406,
+          "text": ","
+        },
+        {
+          "id": 29033,
+          "logprob": -2.2324219,
+          "text": " Gir"
+        },
+        {
+          "id": 1622,
+          "logprob": -0.10797119,
+          "text": "af"
+        },
+        {
+          "id": 249,
+          "logprob": -0.04248047,
+          "text": "at"
+        },
+        {
+          "id": 1480,
+          "logprob": -0.0024814606,
+          "text": "ron"
+        },
+        {
+          "id": 12,
+          "logprob": -1.4277344,
+          "text": "!"
+        },
+        {
+          "id": 193,
+          "logprob": -1.1005859,
+          "text": "\n"
+        },
+        {
+          "id": 50,
+          "logprob": -0.056884766,
+          "text": "G"
+        },
+        {
+          "id": 330,
+          "logprob": -0.1315918,
+          "text": "ir"
+        },
+        {
+          "id": 1622,
+          "logprob": -0.007095337,
+          "text": "af"
+        },
+        {
+          "id": 249,
+          "logprob": -0.00844574,
+          "text": "at"
+        },
+        {
+          "id": 1480,
+          "logprob": -0.00068998337,
+          "text": "ron"
+        },
+        {
+          "id": 37,
+          "logprob": -0.0074768066,
+          "text": ":"
         }
       ],
       "seed": null,
       "tokens": [
+        {
+          "id": 23090,
+          "logprob": -1.8251953,
+          "special": false,
+          "text": " Hello"
+        },
+        {
+          "id": 23,
+          "logprob": -0.31762695,
+          "special": false,
+          "text": ","
+        },
+        {
+          "id": 8156,
+          "logprob": -0.2388916,
+          "special": false,
+          "text": " Daniel"
+        },
+        {
+          "id": 12,
+          "logprob": -0.5698242,
+          "special": false,
+          "text": "!"
+        },
         {
           "id": 193,
-          "logprob": -0.27563477,
+          "logprob": -0.6152344,
           "special": false,
           "text": "\n"
         },
         {
-          "id": 27319,
-          "logprob": -1.1015625,
+          "id": 23626,
+          "logprob": -0.42211914,
           "special": false,
-          "text": "Deep"
+          "text": "Daniel"
         },
         {
-          "id": 2860,
-          "logprob": -0.5292969,
+          "id": 37,
+          "logprob": -0.002336502,
           "special": false,
-          "text": " learning"
+          "text": ":"
         },
         {
-          "id": 304,
-          "logprob": -0.26538086,
+          "id": 1634,
+          "logprob": -2.0605469,
           "special": false,
-          "text": " is"
+          "text": " What"
         },
         {
-          "id": 241,
-          "logprob": -0.41430664,
+          "id": 18,
+          "logprob": -1.5292969,
           "special": false,
-          "text": " a"
+          "text": "'"
         },
         {
-          "id": 30095,
-          "logprob": -1.2724609,
+          "id": 94,
+          "logprob": -0.007926941,
           "special": false,
-          "text": " subset"
-        },
-        {
-          "id": 275,
-          "logprob": -0.0115737915,
-          "special": false,
-          "text": " of"
-        },
-        {
-          "id": 3414,
-          "logprob": -0.2409668,
-          "special": false,
-          "text": " machine"
-        },
-        {
-          "id": 2860,
-          "logprob": -0.008323669,
-          "special": false,
-          "text": " learning"
-        },
-        {
-          "id": 325,
-          "logprob": -1.1533203,
-          "special": false,
-          "text": " that"
+          "text": "s"
         }
       ]
     },
-    "generated_text": "\nDeep learning is a subset of machine learning that"
+    "generated_text": " Hello, Daniel!\nDaniel: What's"
   },
   {
     "details": {
@@ -300,95 +1140,375 @@
       "generated_tokens": 10,
       "prefill": [
         {
-          "id": 1562,
+          "id": 50,
           "logprob": null,
-          "text": "What"
+          "text": "G"
+        },
+        {
+          "id": 330,
+          "logprob": -5.96875,
+          "text": "ir"
+        },
+        {
+          "id": 1622,
+          "logprob": -5.6171875,
+          "text": "af"
+        },
+        {
+          "id": 249,
+          "logprob": -6.5,
+          "text": "at"
+        },
+        {
+          "id": 1480,
+          "logprob": -8.0703125,
+          "text": "ron"
         },
         {
           "id": 304,
-          "logprob": -1.5371094,
+          "logprob": -2.328125,
           "text": " is"
         },
         {
-          "id": 12604,
-          "logprob": -8.7109375,
-          "text": " Deep"
+          "id": 23866,
+          "logprob": -9.59375,
+          "text": " obsessed"
         },
         {
-          "id": 9078,
-          "logprob": -1.9384766,
-          "text": " Learning"
+          "id": 335,
+          "logprob": -0.048339844,
+          "text": " with"
         },
         {
-          "id": 42,
-          "logprob": -0.32958984,
-          "text": "?"
+          "id": 26680,
+          "logprob": -4.0,
+          "text": " gir"
+        },
+        {
+          "id": 1903,
+          "logprob": -0.07531738,
+          "text": "aff"
+        },
+        {
+          "id": 255,
+          "logprob": -0.006793976,
+          "text": "es"
+        },
+        {
+          "id": 23,
+          "logprob": -1.5478516,
+          "text": ","
+        },
+        {
+          "id": 248,
+          "logprob": -4.3320312,
+          "text": " the"
+        },
+        {
+          "id": 758,
+          "logprob": -3.7363281,
+          "text": " most"
+        },
+        {
+          "id": 21735,
+          "logprob": -5.1132812,
+          "text": " glorious"
+        },
+        {
+          "id": 5985,
+          "logprob": -2.0957031,
+          "text": " animal"
+        },
+        {
+          "id": 313,
+          "logprob": -1.1835938,
+          "text": " on"
+        },
+        {
+          "id": 248,
+          "logprob": -0.77685547,
+          "text": " the"
+        },
+        {
+          "id": 1936,
+          "logprob": -2.3808594,
+          "text": " face"
+        },
+        {
+          "id": 275,
+          "logprob": -0.004436493,
+          "text": " of"
+        },
+        {
+          "id": 414,
+          "logprob": -1.9638672,
+          "text": " this"
+        },
+        {
+          "id": 6490,
+          "logprob": -2.0449219,
+          "text": " Earth"
+        },
+        {
+          "id": 25,
+          "logprob": -0.28198242,
+          "text": "."
+        },
+        {
+          "id": 401,
+          "logprob": -7.9179688,
+          "text": " G"
+        },
+        {
+          "id": 6013,
+          "logprob": -2.2734375,
+          "text": "ira"
+        },
+        {
+          "id": 694,
+          "logprob": -0.6230469,
+          "text": "ft"
+        },
+        {
+          "id": 1480,
+          "logprob": -0.20947266,
+          "text": "ron"
+        },
+        {
+          "id": 9369,
+          "logprob": -4.5546875,
+          "text": " believes"
+        },
+        {
+          "id": 455,
+          "logprob": -4.5703125,
+          "text": " all"
+        },
+        {
+          "id": 599,
+          "logprob": -2.7382812,
+          "text": " other"
+        },
+        {
+          "id": 5632,
+          "logprob": -0.21948242,
+          "text": " animals"
+        },
+        {
+          "id": 362,
+          "logprob": -0.7661133,
+          "text": " are"
+        },
+        {
+          "id": 23981,
+          "logprob": -4.9960938,
+          "text": " irrelevant"
+        },
+        {
+          "id": 635,
+          "logprob": -4.234375,
+          "text": " when"
+        },
+        {
+          "id": 4354,
+          "logprob": -0.5131836,
+          "text": " compared"
+        },
+        {
+          "id": 271,
+          "logprob": -0.10357666,
+          "text": " to"
+        },
+        {
+          "id": 248,
+          "logprob": -0.58447266,
+          "text": " the"
+        },
+        {
+          "id": 21735,
+          "logprob": -3.6816406,
+          "text": " glorious"
+        },
+        {
+          "id": 64398,
+          "logprob": -1.8203125,
+          "text": " majesty"
+        },
+        {
+          "id": 275,
+          "logprob": -0.23583984,
+          "text": " of"
+        },
+        {
+          "id": 248,
+          "logprob": -0.35473633,
+          "text": " the"
+        },
+        {
+          "id": 26680,
+          "logprob": -0.24572754,
+          "text": " gir"
+        },
+        {
+          "id": 23226,
+          "logprob": -0.029586792,
+          "text": "affe"
+        },
+        {
+          "id": 25,
+          "logprob": -0.17346191,
+          "text": "."
+        },
+        {
+          "id": 193,
+          "logprob": -1.3945312,
+          "text": "\n"
+        },
+        {
+          "id": 23626,
+          "logprob": -10.0625,
+          "text": "Daniel"
+        },
+        {
+          "id": 37,
+          "logprob": -4.59375,
+          "text": ":"
+        },
+        {
+          "id": 23090,
+          "logprob": -6.9375,
+          "text": " Hello"
+        },
+        {
+          "id": 23,
+          "logprob": -0.99316406,
+          "text": ","
+        },
+        {
+          "id": 29033,
+          "logprob": -2.2324219,
+          "text": " Gir"
+        },
+        {
+          "id": 1622,
+          "logprob": -0.10797119,
+          "text": "af"
+        },
+        {
+          "id": 249,
+          "logprob": -0.04248047,
+          "text": "at"
+        },
+        {
+          "id": 1480,
+          "logprob": -0.0024814606,
+          "text": "ron"
+        },
+        {
+          "id": 12,
+          "logprob": -1.4277344,
+          "text": "!"
+        },
+        {
+          "id": 193,
+          "logprob": -1.1005859,
+          "text": "\n"
+        },
+        {
+          "id": 50,
+          "logprob": -0.056884766,
+          "text": "G"
+        },
+        {
+          "id": 330,
+          "logprob": -0.1315918,
+          "text": "ir"
+        },
+        {
+          "id": 1622,
+          "logprob": -0.007095337,
+          "text": "af"
+        },
+        {
+          "id": 249,
+          "logprob": -0.00844574,
+          "text": "at"
+        },
+        {
+          "id": 1480,
+          "logprob": -0.00068998337,
+          "text": "ron"
+        },
+        {
+          "id": 37,
+          "logprob": -0.0074768066,
+          "text": ":"
         }
       ],
       "seed": null,
       "tokens": [
+        {
+          "id": 23090,
+          "logprob": -1.8251953,
+          "special": false,
+          "text": " Hello"
+        },
+        {
+          "id": 23,
+          "logprob": -0.31762695,
+          "special": false,
+          "text": ","
+        },
+        {
+          "id": 8156,
+          "logprob": -0.2388916,
+          "special": false,
+          "text": " Daniel"
+        },
+        {
+          "id": 12,
+          "logprob": -0.5698242,
+          "special": false,
+          "text": "!"
+        },
         {
           "id": 193,
-          "logprob": -0.27563477,
+          "logprob": -0.6152344,
           "special": false,
           "text": "\n"
         },
         {
-          "id": 27319,
-          "logprob": -1.1015625,
+          "id": 23626,
+          "logprob": -0.42211914,
           "special": false,
-          "text": "Deep"
+          "text": "Daniel"
         },
         {
-          "id": 2860,
-          "logprob": -0.5292969,
+          "id": 37,
+          "logprob": -0.002336502,
           "special": false,
-          "text": " learning"
+          "text": ":"
         },
         {
-          "id": 304,
-          "logprob": -0.26538086,
+          "id": 1634,
+          "logprob": -2.0605469,
           "special": false,
-          "text": " is"
+          "text": " What"
         },
         {
-          "id": 241,
-          "logprob": -0.41430664,
+          "id": 18,
+          "logprob": -1.5292969,
           "special": false,
-          "text": " a"
+          "text": "'"
         },
         {
-          "id": 30095,
-          "logprob": -1.2724609,
+          "id": 94,
+          "logprob": -0.007926941,
           "special": false,
-          "text": " subset"
-        },
-        {
-          "id": 275,
-          "logprob": -0.0115737915,
-          "special": false,
-          "text": " of"
-        },
-        {
-          "id": 3414,
-          "logprob": -0.2409668,
-          "special": false,
-          "text": " machine"
-        },
-        {
-          "id": 2860,
-          "logprob": -0.008323669,
-          "special": false,
-          "text": " learning"
-        },
-        {
-          "id": 325,
-          "logprob": -1.1533203,
-          "special": false,
-          "text": " that"
+          "text": "s"
         }
       ]
     },
-    "generated_text": "\nDeep learning is a subset of machine learning that"
+    "generated_text": " Hello, Daniel!\nDaniel: What's"
   }
 ]
diff --git a/integration-tests/models/__snapshots__/test_flash_llama/test_flash_llama_all_params.json b/integration-tests/models/__snapshots__/test_flash_llama/test_flash_llama_all_params.json
index bc12470e..9f145377 100644
--- a/integration-tests/models/__snapshots__/test_flash_llama/test_flash_llama_all_params.json
+++ b/integration-tests/models/__snapshots__/test_flash_llama/test_flash_llama_all_params.json
@@ -1,8 +1,8 @@
 {
   "details": {
     "best_of_sequences": null,
-    "finish_reason": "length",
-    "generated_tokens": 10,
+    "finish_reason": "stop_sequence",
+    "generated_tokens": 5,
     "prefill": [
       {
         "id": 1,
@@ -28,61 +28,32 @@
         "special": false,
         "text": " failed"
       },
-      {
-        "id": 363,
-        "logprob": -1.1064453,
-        "special": false,
-        "text": " for"
-      },
-      {
-        "id": 278,
-        "logprob": -0.7709961,
-        "special": false,
-        "text": " the"
-      },
-      {
-        "id": 1494,
-        "logprob": -0.13464355,
-        "special": false,
-        "text": " following"
-      },
-      {
-        "id": 9590,
-        "logprob": -0.1986084,
-        "special": false,
-        "text": " reasons"
-      },
       {
         "id": 29901,
-        "logprob": 0.0,
+        "logprob": -0.44970703,
         "special": false,
         "text": ":"
       },
       {
-        "id": 13,
+        "id": 4829,
+        "logprob": -1.8339844,
+        "special": false,
+        "text": " Error"
+      },
+      {
+        "id": 297,
+        "logprob": -1.0556641,
+        "special": false,
+        "text": " in"
+      },
+      {
+        "id": 1243,
         "logprob": 0.0,
         "special": false,
-        "text": "\n"
-      },
-      {
-        "id": 1576,
-        "logprob": -0.6166992,
-        "special": false,
-        "text": "The"
-      },
-      {
-        "id": 13877,
-        "logprob": -0.91845703,
-        "special": false,
-        "text": " requested"
-      },
-      {
-        "id": 3988,
-        "logprob": -0.7680664,
-        "special": false,
-        "text": " URL"
+        "text": " test"
       }
-    ]
+    ],
+    "top_tokens": null
   },
-  "generated_text": "Test request failed for the following reasons:\nThe requested URL"
+  "generated_text": "Test request failed: Error in test"
 }
diff --git a/integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq_all_params.json b/integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq_all_params.json
index 8d705c59..02713a00 100644
--- a/integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq_all_params.json
+++ b/integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq_all_params.json
@@ -11,12 +11,12 @@
       },
       {
         "id": 4321,
-        "logprob": -9.5625,
+        "logprob": -9.6015625,
         "text": "Test"
       },
       {
         "id": 2009,
-        "logprob": -9.6796875,
+        "logprob": -9.6640625,
         "text": "request"
       }
     ],
@@ -24,33 +24,15 @@
     "tokens": [
       {
         "id": 29899,
-        "logprob": -1.1972656,
+        "logprob": -1.1640625,
         "special": false,
         "text": "-"
       },
       {
-        "id": 29896,
-        "logprob": -1.1621094,
+        "id": 1454,
+        "logprob": -0.07543945,
         "special": false,
-        "text": "1"
-      },
-      {
-        "id": 13,
-        "logprob": -0.12768555,
-        "special": false,
-        "text": "\n"
-      },
-      {
-        "id": 3057,
-        "logprob": -0.074279785,
-        "special": false,
-        "text": "Test"
-      },
-      {
-        "id": 2009,
-        "logprob": 0.0,
-        "special": false,
-        "text": " request"
+        "text": "for"
       },
       {
         "id": 29899,
@@ -58,31 +40,49 @@
         "special": false,
         "text": "-"
       },
+      {
+        "id": 9342,
+        "logprob": 0.0,
+        "special": false,
+        "text": "comment"
+      },
+      {
+        "id": 29901,
+        "logprob": 0.0,
+        "special": false,
+        "text": ":"
+      },
+      {
+        "id": 396,
+        "logprob": -0.2956543,
+        "special": false,
+        "text": " #"
+      },
       {
         "id": 29906,
-        "logprob": -0.20141602,
+        "logprob": -0.52734375,
         "special": false,
         "text": "2"
       },
       {
-        "id": 13,
-        "logprob": 0.0,
+        "id": 29900,
+        "logprob": -0.6899414,
         "special": false,
-        "text": "\n"
+        "text": "0"
       },
       {
-        "id": 3057,
-        "logprob": -0.6611328,
-        "special": false,
-        "text": "Test"
-      },
-      {
-        "id": 2009,
+        "id": 29896,
         "logprob": 0.0,
         "special": false,
-        "text": " request"
+        "text": "1"
+      },
+      {
+        "id": 29946,
+        "logprob": -1.5068359,
+        "special": false,
+        "text": "4"
       }
     ]
   },
-  "generated_text": "Test request-1\nTest request-2\nTest request"
+  "generated_text": "Test request-for-comment: #2014"
 }
diff --git a/integration-tests/models/__snapshots__/test_flash_mistral/test_flash_mistral_all_params.json b/integration-tests/models/__snapshots__/test_flash_mistral/test_flash_mistral_all_params.json
index 91ac444e..c0dc6471 100644
--- a/integration-tests/models/__snapshots__/test_flash_mistral/test_flash_mistral_all_params.json
+++ b/integration-tests/models/__snapshots__/test_flash_mistral/test_flash_mistral_all_params.json
@@ -35,10 +35,10 @@
         "text": " Let"
       },
       {
-        "id": 268,
-        "logprob": -2.0234375,
+        "id": 332,
+        "logprob": -2.3359375,
         "special": false,
-        "text": " s"
+        "text": " u"
       },
       {
         "id": 347,
@@ -46,43 +46,44 @@
         "special": false,
         "text": " be"
       },
+      {
+        "id": 325,
+        "logprob": -1.0234375,
+        "special": false,
+        "text": " ("
+      },
+      {
+        "id": 28734,
+        "logprob": -2.0292969,
+        "special": false,
+        "text": "0"
+      },
+      {
+        "id": 648,
+        "logprob": -1.0439453,
+        "special": false,
+        "text": " +"
+      },
       {
         "id": 28705,
-        "logprob": -0.44458008,
+        "logprob": -0.24499512,
         "special": false,
         "text": " "
       },
       {
-        "id": 28784,
-        "logprob": -0.94189453,
+        "id": 28770,
+        "logprob": -0.5073242,
         "special": false,
-        "text": "6"
+        "text": "3"
       },
       {
-        "id": 28748,
-        "logprob": 0.0,
+        "id": 387,
+        "logprob": -1.5507812,
         "special": false,
-        "text": "/"
-      },
-      {
-        "id": 6422,
-        "logprob": 0.0,
-        "special": false,
-        "text": "(-"
-      },
-      {
-        "id": 28783,
-        "logprob": -0.2590332,
-        "special": false,
-        "text": "8"
-      },
-      {
-        "id": 28731,
-        "logprob": -1.0039062,
-        "special": false,
-        "text": ")"
+        "text": " -"
       }
-    ]
+    ],
+    "top_tokens": null
   },
-  "generated_text": "Test request: Let s be 6/(-8)"
+  "generated_text": "Test request: Let u be (0 + 3 -"
 }
diff --git a/integration-tests/models/__snapshots__/test_flash_starcoder/test_flash_starcoder_default_params.json b/integration-tests/models/__snapshots__/test_flash_starcoder/test_flash_starcoder_default_params.json
index 86d55a17..89e02c07 100644
--- a/integration-tests/models/__snapshots__/test_flash_starcoder/test_flash_starcoder_default_params.json
+++ b/integration-tests/models/__snapshots__/test_flash_starcoder/test_flash_starcoder_default_params.json
@@ -11,7 +11,7 @@
       },
       {
         "id": 1459,
-        "logprob": -5.6289062,
+        "logprob": -5.6328125,
         "text": " print"
       },
       {
@@ -21,7 +21,7 @@
       },
       {
         "id": 7656,
-        "logprob": -5.9960938,
+        "logprob": -5.9882812,
         "text": "hello"
       }
     ],
@@ -59,13 +59,13 @@
       },
       {
         "id": 10896,
-        "logprob": -0.3659668,
+        "logprob": -0.38549805,
         "special": false,
         "text": " World"
       },
       {
         "id": 657,
-        "logprob": -0.49804688,
+        "logprob": -0.5229492,
         "special": false,
         "text": "\")"
       },
@@ -113,7 +113,7 @@
       },
       {
         "id": 426,
-        "logprob": -0.051635742,
+        "logprob": 0.0,
         "special": false,
         "text": "name"
       },
@@ -148,22 +148,10 @@
         "text": " print"
       },
       {
-        "id": 26,
-        "logprob": -1.9101562,
+        "id": 440,
+        "logprob": -0.16027832,
         "special": false,
-        "text": "("
-      },
-      {
-        "id": 88,
-        "logprob": 0.0,
-        "special": false,
-        "text": "f"
-      },
-      {
-        "id": 20,
-        "logprob": 0.0,
-        "special": false,
-        "text": "\""
+        "text": "(\""
       },
       {
         "id": 8279,
@@ -172,22 +160,28 @@
         "text": "Hello"
       },
       {
-        "id": 301,
+        "id": 313,
         "logprob": 0.0,
         "special": false,
-        "text": " {"
+        "text": " \""
       },
       {
-        "id": 426,
+        "id": 474,
         "logprob": 0.0,
         "special": false,
-        "text": "name"
+        "text": " +"
       },
       {
-        "id": 8474,
+        "id": 636,
         "logprob": 0.0,
         "special": false,
-        "text": "}\")"
+        "text": " name"
+      },
+      {
+        "id": 27,
+        "logprob": 0.0,
+        "special": false,
+        "text": ")"
       },
       {
         "id": 203,
@@ -292,22 +286,10 @@
         "text": " print"
       },
       {
-        "id": 26,
+        "id": 440,
         "logprob": 0.0,
         "special": false,
-        "text": "("
-      },
-      {
-        "id": 88,
-        "logprob": 0.0,
-        "special": false,
-        "text": "f"
-      },
-      {
-        "id": 20,
-        "logprob": 0.0,
-        "special": false,
-        "text": "\""
+        "text": "(\""
       },
       {
         "id": 8279,
@@ -316,40 +298,58 @@
         "text": "Hello"
       },
       {
-        "id": 301,
+        "id": 313,
         "logprob": 0.0,
         "special": false,
-        "text": " {"
+        "text": " \""
       },
       {
-        "id": 426,
+        "id": 474,
         "logprob": 0.0,
         "special": false,
-        "text": "name"
+        "text": " +"
       },
       {
-        "id": 835,
-        "logprob": -0.4074707,
-        "special": false,
-        "text": "},"
-      },
-      {
-        "id": 844,
+        "id": 636,
         "logprob": 0.0,
         "special": false,
-        "text": " you"
+        "text": " name"
       },
       {
-        "id": 884,
+        "id": 474,
         "logprob": 0.0,
         "special": false,
-        "text": " are"
+        "text": " +"
       },
       {
-        "id": 301,
+        "id": 313,
+        "logprob": -0.6328125,
+        "special": false,
+        "text": " \""
+      },
+      {
+        "id": 313,
+        "logprob": -1.7011719,
+        "special": false,
+        "text": " \""
+      },
+      {
+        "id": 474,
         "logprob": 0.0,
         "special": false,
-        "text": " {"
+        "text": " +"
+      },
+      {
+        "id": 596,
+        "logprob": 0.0,
+        "special": false,
+        "text": " str"
+      },
+      {
+        "id": 26,
+        "logprob": 0.0,
+        "special": false,
+        "text": "("
       },
       {
         "id": 381,
@@ -358,36 +358,36 @@
         "text": "age"
       },
       {
-        "id": 111,
+        "id": 490,
         "logprob": 0.0,
         "special": false,
-        "text": "}"
-      },
-      {
-        "id": 11274,
-        "logprob": 0.0,
-        "special": false,
-        "text": " years"
-      },
-      {
-        "id": 3610,
-        "logprob": 0.0,
-        "special": false,
-        "text": " old"
-      },
-      {
-        "id": 657,
-        "logprob": 0.0,
-        "special": false,
-        "text": "\")"
+        "text": "))"
       },
       {
         "id": 203,
         "logprob": 0.0,
         "special": false,
         "text": "\n"
+      },
+      {
+        "id": 203,
+        "logprob": 0.0,
+        "special": false,
+        "text": "\n"
+      },
+      {
+        "id": 589,
+        "logprob": 0.0,
+        "special": false,
+        "text": "def"
+      },
+      {
+        "id": 1459,
+        "logprob": 0.0,
+        "special": false,
+        "text": " print"
       }
     ]
   },
-  "generated_text": "():\n    print(\"Hello World\")\n\ndef print_hello_name(name):\n    print(f\"Hello {name}\")\n\ndef print_hello_name_age(name, age):\n    print(f\"Hello {name}, you are {age} years old\")\n"
+  "generated_text": "():\n    print(\"Hello World\")\n\ndef print_hello_name(name):\n    print(\"Hello \" + name)\n\ndef print_hello_name_age(name, age):\n    print(\"Hello \" + name + \" \" + str(age))\n\ndef print"
 }
diff --git a/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq_default_params.json b/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq_default_params.json
index 3815a2ef..5598a2ad 100644
--- a/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq_default_params.json
+++ b/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq_default_params.json
@@ -11,7 +11,7 @@
       },
       {
         "id": 3226,
-        "logprob": -9.015625,
+        "logprob": -9.0234375,
         "text": " ge"
       },
       {
@@ -21,47 +21,47 @@
       },
       {
         "id": 81,
-        "logprob": -0.25634766,
+        "logprob": -0.25976562,
         "text": "_"
       },
       {
         "id": 6009,
-        "logprob": -2.203125,
+        "logprob": -2.2148438,
         "text": "mean"
       },
       {
         "id": 26,
-        "logprob": -0.30322266,
+        "logprob": -0.3010254,
         "text": "("
       },
       {
         "id": 62,
-        "logprob": -5.6015625,
+        "logprob": -5.6757812,
         "text": "L"
       },
       {
         "id": 44,
-        "logprob": -3.0878906,
+        "logprob": -3.0898438,
         "text": ":"
       },
       {
         "id": 1682,
-        "logprob": -0.6826172,
+        "logprob": -0.6791992,
         "text": " List"
       },
       {
         "id": 77,
-        "logprob": -0.38354492,
+        "logprob": -0.38891602,
         "text": "["
       },
       {
         "id": 1808,
-        "logprob": -0.9760742,
+        "logprob": -0.92041016,
         "text": "float"
       },
       {
         "id": 10794,
-        "logprob": -2.5234375,
+        "logprob": -2.5390625,
         "text": "]):"
       }
     ],
@@ -81,7 +81,7 @@
       },
       {
         "id": 11665,
-        "logprob": -1.0537109,
+        "logprob": -1.6005859,
         "special": false,
         "text": " reduce"
       },
@@ -159,7 +159,7 @@
       },
       {
         "id": 203,
-        "logprob": -0.10021973,
+        "logprob": -0.11968994,
         "special": false,
         "text": "\n"
       },