From 0b95693fb8b9640283a0fcf40ac4dd2ab15187eb Mon Sep 17 00:00:00 2001
From: drbh <david.richard.holtz@gmail.com>
Date: Mon, 29 Jul 2024 11:38:38 -0400
Subject: [PATCH] fix: adjust test snapshots and small refactors (#2323)

* fix: adjust test snapshots and small refactors

* fix: revert non snapshot changes
---
 .../test_bloom_560m/test_bloom_560m.json      |  85 +++++-----
 .../test_bloom_560m_all_params.json           |  55 ++++---
 .../test_bloom_560m_sharded.json              |  85 +++++-----
 ...t_flash_llama_completion_many_prompts.json |  18 +-
 .../test_flash_deepseek_v2_all_params.json    |  54 +-----
 .../test_flash_deepseek_v2_load.json          | 154 +++++++++---------
 .../test_flash_gemma/test_flash_gemma.json    |  34 ++--
 .../test_flash_gemma_all_params.json          |  20 +--
 .../test_flash_gemma_gptq_all_params.json     |  84 +++++-----
 .../test_flash_starcoder_default_params.json  |  17 +-
 .../test_flash_starcoder2_default_params.json | 120 +++++++-------
 .../__snapshots__/test_mamba/test_mamba.json  |  18 +-
 .../test_mamba/test_mamba_all_params.json     |   6 +-
 13 files changed, 359 insertions(+), 391 deletions(-)

diff --git a/integration-tests/models/__snapshots__/test_bloom_560m/test_bloom_560m.json b/integration-tests/models/__snapshots__/test_bloom_560m/test_bloom_560m.json
index 53a4ab85..b274992e 100644
--- a/integration-tests/models/__snapshots__/test_bloom_560m/test_bloom_560m.json
+++ b/integration-tests/models/__snapshots__/test_bloom_560m/test_bloom_560m.json
@@ -11,52 +11,52 @@
       },
       {
         "id": 49833,
-        "logprob": -10.5625,
+        "logprob": -10.5703125,
         "text": " dég"
       },
       {
         "id": 21543,
-        "logprob": -0.14770508,
+        "logprob": -0.14746094,
         "text": "uster"
       },
       {
         "id": 447,
-        "logprob": -1.9287109,
+        "logprob": -1.9277344,
         "text": " un"
       },
       {
         "id": 46341,
-        "logprob": -15.4609375,
+        "logprob": -15.421875,
         "text": " ort"
       },
       {
         "id": 35567,
-        "logprob": -7.5585938,
+        "logprob": -7.5820312,
         "text": "olan"
       },
       {
         "id": 15,
-        "logprob": -1.4003906,
+        "logprob": -1.4013672,
         "text": ","
       },
       {
         "id": 1669,
-        "logprob": -1.5673828,
+        "logprob": -1.5664062,
         "text": " il"
       },
       {
         "id": 11580,
-        "logprob": -0.94628906,
+        "logprob": -0.94189453,
         "text": " faut"
       },
       {
         "id": 3913,
-        "logprob": -3.703125,
+        "logprob": -3.6816406,
         "text": " tout"
       },
       {
         "id": 39261,
-        "logprob": -1.5732422,
+        "logprob": -1.7753906,
         "text": " d'abord"
       }
     ],
@@ -64,65 +64,66 @@
     "tokens": [
       {
         "id": 578,
-        "logprob": -1.6591797,
+        "logprob": -1.6318359,
         "special": false,
         "text": " le"
       },
       {
         "id": 5608,
-        "logprob": -2.4492188,
+        "logprob": -2.4882812,
         "special": false,
         "text": " faire"
       },
       {
-        "id": 159570,
-        "logprob": -6.6835938,
+        "id": 7735,
+        "logprob": -2.4355469,
         "special": false,
-        "text": " réch"
+        "text": " fond"
       },
       {
-        "id": 810,
+        "id": 289,
         "logprob": 0.0,
         "special": false,
-        "text": "au"
+        "text": "re"
       },
       {
-        "id": 12736,
+        "id": 693,
+        "logprob": -2.4472656,
+        "special": false,
+        "text": " à"
+      },
+      {
+        "id": 366,
+        "logprob": -1.1972656,
+        "special": false,
+        "text": " la"
+      },
+      {
+        "id": 48844,
+        "logprob": -1.7890625,
+        "special": false,
+        "text": " cass"
+      },
+      {
+        "id": 1744,
         "logprob": 0.0,
         "special": false,
-        "text": "ffer"
+        "text": "ero"
       },
       {
-        "id": 1742,
-        "logprob": -2.5175781,
-        "special": false,
-        "text": " au"
-      },
-      {
-        "id": 6105,
-        "logprob": -2.0078125,
-        "special": false,
-        "text": " bain"
-      },
-      {
-        "id": 88254,
-        "logprob": -0.12695312,
-        "special": false,
-        "text": "-mar"
-      },
-      {
-        "id": 641,
+        "id": 327,
         "logprob": 0.0,
         "special": false,
-        "text": "ie"
+        "text": "le"
       },
       {
         "id": 2940,
-        "logprob": -3.5175781,
+        "logprob": -1.9335938,
         "special": false,
         "text": " avec"
       }
-    ]
+    ],
+    "top_tokens": null
   },
-  "generated_text": " le faire réchauffer au bain-marie avec"
+  "generated_text": " le faire fondre à la casserole avec"
 }
diff --git a/integration-tests/models/__snapshots__/test_bloom_560m/test_bloom_560m_all_params.json b/integration-tests/models/__snapshots__/test_bloom_560m/test_bloom_560m_all_params.json
index ace73416..9422f27f 100644
--- a/integration-tests/models/__snapshots__/test_bloom_560m/test_bloom_560m_all_params.json
+++ b/integration-tests/models/__snapshots__/test_bloom_560m/test_bloom_560m_all_params.json
@@ -11,7 +11,7 @@
       },
       {
         "id": 1669,
-        "logprob": -5.4414062,
+        "logprob": -5.4453125,
         "text": " il"
       },
       {
@@ -21,12 +21,12 @@
       },
       {
         "id": 3913,
-        "logprob": -4.3554688,
+        "logprob": -4.3320312,
         "text": " tout"
       },
       {
         "id": 39261,
-        "logprob": -2.9238281,
+        "logprob": -2.9160156,
         "text": " d'abord"
       }
     ],
@@ -34,65 +34,66 @@
     "tokens": [
       {
         "id": 408,
-        "logprob": -0.07891846,
+        "logprob": -0.16687012,
         "special": false,
         "text": " que"
       },
       {
         "id": 366,
-        "logprob": -1.2939453,
+        "logprob": -1.5517578,
         "special": false,
         "text": " la"
       },
       {
         "id": 8769,
-        "logprob": -0.3708496,
+        "logprob": -0.16687012,
         "special": false,
         "text": " personne"
       },
       {
         "id": 1479,
-        "logprob": -2.2871094,
+        "logprob": -2.1035156,
         "special": false,
         "text": " qui"
       },
       {
-        "id": 2997,
-        "logprob": -0.8671875,
+        "id": 143926,
+        "logprob": -2.8671875,
         "special": false,
-        "text": " vous"
+        "text": " réalise"
       },
       {
-        "id": 35977,
-        "logprob": -1.5097656,
+        "id": 578,
+        "logprob": 0.0,
         "special": false,
-        "text": " suit"
+        "text": " le"
       },
       {
-        "id": 21558,
-        "logprob": -0.07891846,
+        "id": 8138,
+        "logprob": -0.66748047,
         "special": false,
-        "text": " ait"
+        "text": " projet"
       },
       {
-        "id": 447,
-        "logprob": -0.12695312,
+        "id": 795,
+        "logprob": -1.6279297,
         "special": false,
-        "text": " un"
+        "text": " ne"
       },
       {
-        "id": 78606,
-        "logprob": -2.21875,
+        "id": 9802,
+        "logprob": -0.47875977,
         "special": false,
-        "text": " profil"
+        "text": " soit"
       },
       {
-        "id": 3899,
-        "logprob": -1.3535156,
+        "id": 1230,
+        "logprob": 0.0,
         "special": false,
-        "text": " bien"
+        "text": " pas"
       }
-    ]
+    ],
+    "top_tokens": null
   },
-  "generated_text": "Pour déguster un ortolan, il faut tout d'abord que la personne qui vous suit ait un profil bien"
+  "generated_text": "Pour déguster un ortolan, il faut tout d'abord que la personne qui réalise le projet ne soit pas"
 }
diff --git a/integration-tests/models/__snapshots__/test_bloom_560m_sharded/test_bloom_560m_sharded.json b/integration-tests/models/__snapshots__/test_bloom_560m_sharded/test_bloom_560m_sharded.json
index dd8936af..b17c889e 100644
--- a/integration-tests/models/__snapshots__/test_bloom_560m_sharded/test_bloom_560m_sharded.json
+++ b/integration-tests/models/__snapshots__/test_bloom_560m_sharded/test_bloom_560m_sharded.json
@@ -11,52 +11,52 @@
       },
       {
         "id": 49833,
-        "logprob": -10.5390625,
+        "logprob": -10.546875,
         "text": " dég"
       },
       {
         "id": 21543,
-        "logprob": -0.14758301,
+        "logprob": -0.14819336,
         "text": "uster"
       },
       {
         "id": 447,
-        "logprob": -1.9296875,
+        "logprob": -1.9257812,
         "text": " un"
       },
       {
         "id": 46341,
-        "logprob": -15.4453125,
+        "logprob": -15.4296875,
         "text": " ort"
       },
       {
         "id": 35567,
-        "logprob": -7.59375,
+        "logprob": -7.5625,
         "text": "olan"
       },
       {
         "id": 15,
-        "logprob": -1.3994141,
+        "logprob": -1.4199219,
         "text": ","
       },
       {
         "id": 1669,
-        "logprob": -1.578125,
+        "logprob": -1.5634766,
         "text": " il"
       },
       {
         "id": 11580,
-        "logprob": -0.9453125,
+        "logprob": -0.9458008,
         "text": " faut"
       },
       {
         "id": 3913,
-        "logprob": -3.7011719,
+        "logprob": -3.6816406,
         "text": " tout"
       },
       {
         "id": 39261,
-        "logprob": -1.5732422,
+        "logprob": -1.7753906,
         "text": " d'abord"
       }
     ],
@@ -64,65 +64,66 @@
     "tokens": [
       {
         "id": 578,
-        "logprob": -1.6474609,
+        "logprob": -1.828125,
         "special": false,
         "text": " le"
       },
       {
         "id": 5608,
-        "logprob": -2.5097656,
+        "logprob": -2.5546875,
         "special": false,
         "text": " faire"
       },
       {
-        "id": 159570,
-        "logprob": -6.65625,
+        "id": 7735,
+        "logprob": -2.4277344,
         "special": false,
-        "text": " réch"
+        "text": " fond"
       },
       {
-        "id": 810,
+        "id": 289,
         "logprob": 0.0,
         "special": false,
-        "text": "au"
+        "text": "re"
       },
       {
-        "id": 12736,
+        "id": 693,
+        "logprob": -2.4472656,
+        "special": false,
+        "text": " à"
+      },
+      {
+        "id": 366,
+        "logprob": -1.1494141,
+        "special": false,
+        "text": " la"
+      },
+      {
+        "id": 48844,
+        "logprob": -1.7939453,
+        "special": false,
+        "text": " cass"
+      },
+      {
+        "id": 1744,
         "logprob": 0.0,
         "special": false,
-        "text": "ffer"
+        "text": "ero"
       },
       {
-        "id": 1742,
-        "logprob": -2.5859375,
-        "special": false,
-        "text": " au"
-      },
-      {
-        "id": 6105,
-        "logprob": -2.03125,
-        "special": false,
-        "text": " bain"
-      },
-      {
-        "id": 88254,
-        "logprob": -0.12695312,
-        "special": false,
-        "text": "-mar"
-      },
-      {
-        "id": 641,
+        "id": 327,
         "logprob": 0.0,
         "special": false,
-        "text": "ie"
+        "text": "le"
       },
       {
         "id": 2940,
-        "logprob": -3.5175781,
+        "logprob": -1.9013672,
         "special": false,
         "text": " avec"
       }
-    ]
+    ],
+    "top_tokens": null
   },
-  "generated_text": " le faire réchauffer au bain-marie avec"
+  "generated_text": " le faire fondre à la casserole avec"
 }
diff --git a/integration-tests/models/__snapshots__/test_completion_prompts/test_flash_llama_completion_many_prompts.json b/integration-tests/models/__snapshots__/test_completion_prompts/test_flash_llama_completion_many_prompts.json
index 99c33cf7..9f3faffc 100644
--- a/integration-tests/models/__snapshots__/test_completion_prompts/test_flash_llama_completion_many_prompts.json
+++ b/integration-tests/models/__snapshots__/test_completion_prompts/test_flash_llama_completion_many_prompts.json
@@ -1,11 +1,17 @@
 {
   "choices": [
     {
-      "finish_reason": "eos_token",
+      "finish_reason": "stop",
       "index": 1,
       "logprobs": null,
       "text": " PR for more information?"
     },
+    {
+      "finish_reason": "length",
+      "index": 3,
+      "logprobs": null,
+      "text": "hd20220811-"
+    },
     {
       "finish_reason": "length",
       "index": 0,
@@ -17,19 +23,13 @@
       "index": 2,
       "logprobs": null,
       "text": " severely flawed and often has a substandard"
-    },
-    {
-      "finish_reason": "length",
-      "index": 3,
-      "logprobs": null,
-      "text": "hd20220811-"
     }
   ],
-  "created": 1713284455,
+  "created": 1722014725,
   "id": "",
   "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
   "object": "text_completion",
-  "system_fingerprint": "2.0.1-native",
+  "system_fingerprint": "2.2.1-dev0-native",
   "usage": {
     "completion_tokens": 36,
     "prompt_tokens": 8,
diff --git a/integration-tests/models/__snapshots__/test_flash_deepseek_v2/test_flash_deepseek_v2_all_params.json b/integration-tests/models/__snapshots__/test_flash_deepseek_v2/test_flash_deepseek_v2_all_params.json
index e84135cf..6b45cf6b 100644
--- a/integration-tests/models/__snapshots__/test_flash_deepseek_v2/test_flash_deepseek_v2_all_params.json
+++ b/integration-tests/models/__snapshots__/test_flash_deepseek_v2/test_flash_deepseek_v2_all_params.json
@@ -1,8 +1,8 @@
 {
   "details": {
     "best_of_sequences": null,
-    "finish_reason": "length",
-    "generated_tokens": 10,
+    "finish_reason": "eos_token",
+    "generated_tokens": 4,
     "prefill": [
       {
         "id": 100000,
@@ -16,7 +16,7 @@
       },
       {
         "id": 3102,
-        "logprob": -11.1875,
+        "logprob": -11.25,
         "text": " request"
       }
     ],
@@ -30,7 +30,7 @@
       },
       {
         "id": 10081,
-        "logprob": -0.36914062,
+        "logprob": -0.41210938,
         "special": false,
         "text": " successfully"
       },
@@ -41,49 +41,13 @@
         "text": "."
       },
       {
-        "id": 185,
-        "logprob": 0.0,
-        "special": false,
-        "text": "\n"
-      },
-      {
-        "id": 1380,
-        "logprob": -0.38671875,
-        "special": false,
-        "text": "We"
-      },
-      {
-        "id": 543,
-        "logprob": -0.12695312,
-        "special": false,
-        "text": " will"
-      },
-      {
-        "id": 752,
-        "logprob": -0.20117188,
-        "special": false,
-        "text": " get"
-      },
-      {
-        "id": 279,
-        "logprob": 0.0,
-        "special": false,
-        "text": " in"
-      },
-      {
-        "id": 5402,
-        "logprob": 0.0,
-        "special": false,
-        "text": " touch"
-      },
-      {
-        "id": 366,
-        "logprob": 0.0,
-        "special": false,
-        "text": " with"
+        "id": 100001,
+        "logprob": -0.16015625,
+        "special": true,
+        "text": "<｜end▁of▁sentence｜>"
       }
     ],
     "top_tokens": null
   },
-  "generated_text": "Test request sent successfully.\nWe will get in touch with"
+  "generated_text": "Test request sent successfully."
 }
diff --git a/integration-tests/models/__snapshots__/test_flash_deepseek_v2/test_flash_deepseek_v2_load.json b/integration-tests/models/__snapshots__/test_flash_deepseek_v2/test_flash_deepseek_v2_load.json
index a4b9784a..e365829a 100644
--- a/integration-tests/models/__snapshots__/test_flash_deepseek_v2/test_flash_deepseek_v2_load.json
+++ b/integration-tests/models/__snapshots__/test_flash_deepseek_v2/test_flash_deepseek_v2_load.json
@@ -17,7 +17,7 @@
         },
         {
           "id": 3102,
-          "logprob": -11.1875,
+          "logprob": -11.25,
           "text": " request"
         }
       ],
@@ -25,68 +25,68 @@
       "tokens": [
         {
           "id": 185,
-          "logprob": -1.5546875,
+          "logprob": -1.546875,
           "special": false,
           "text": "\n"
         },
         {
           "id": 549,
-          "logprob": -2.8125,
+          "logprob": -2.859375,
           "special": false,
           "text": "The"
         },
         {
           "id": 1727,
-          "logprob": -2.375,
+          "logprob": -2.359375,
           "special": false,
           "text": " test"
         },
         {
           "id": 3102,
-          "logprob": -0.890625,
+          "logprob": -0.83203125,
           "special": false,
           "text": " request"
         },
         {
           "id": 317,
-          "logprob": -1.1484375,
+          "logprob": -1.125,
           "special": false,
           "text": " is"
         },
         {
           "id": 245,
-          "logprob": -1.5390625,
+          "logprob": -1.5703125,
           "special": false,
           "text": " a"
         },
         {
-          "id": 3102,
-          "logprob": -2.609375,
+          "id": 3412,
+          "logprob": -2.578125,
           "special": false,
-          "text": " request"
+          "text": " document"
         },
         {
-          "id": 327,
-          "logprob": -0.75,
+          "id": 344,
+          "logprob": -1.125,
           "special": false,
-          "text": " for"
+          "text": " that"
         },
         {
-          "id": 245,
-          "logprob": -1.1171875,
+          "id": 317,
+          "logprob": -1.6953125,
           "special": false,
-          "text": " a"
+          "text": " is"
         },
         {
-          "id": 1727,
-          "logprob": -0.90625,
+          "id": 1222,
+          "logprob": -1.75,
           "special": false,
-          "text": " test"
+          "text": " used"
         }
       ],
       "top_tokens": null
     },
-    "generated_text": "\nThe test request is a request for a test"
+    "generated_text": "\nThe test request is a document that is used"
   },
   {
     "details": {
@@ -114,68 +114,68 @@
       "tokens": [
         {
           "id": 185,
-          "logprob": -1.5546875,
+          "logprob": -1.546875,
           "special": false,
           "text": "\n"
         },
         {
           "id": 549,
-          "logprob": -2.8125,
+          "logprob": -2.859375,
           "special": false,
           "text": "The"
         },
         {
           "id": 1727,
-          "logprob": -2.375,
+          "logprob": -2.359375,
           "special": false,
           "text": " test"
         },
         {
           "id": 3102,
-          "logprob": -0.890625,
+          "logprob": -0.83203125,
           "special": false,
           "text": " request"
         },
         {
           "id": 317,
-          "logprob": -1.1484375,
+          "logprob": -1.125,
           "special": false,
           "text": " is"
         },
         {
           "id": 245,
-          "logprob": -1.5390625,
+          "logprob": -1.5703125,
           "special": false,
           "text": " a"
         },
         {
-          "id": 3102,
-          "logprob": -2.609375,
+          "id": 3412,
+          "logprob": -2.578125,
           "special": false,
-          "text": " request"
+          "text": " document"
         },
         {
-          "id": 327,
-          "logprob": -0.75,
+          "id": 344,
+          "logprob": -1.125,
           "special": false,
-          "text": " for"
+          "text": " that"
         },
         {
-          "id": 245,
-          "logprob": -1.1171875,
+          "id": 317,
+          "logprob": -1.6953125,
           "special": false,
-          "text": " a"
+          "text": " is"
         },
         {
-          "id": 1727,
-          "logprob": -0.90625,
+          "id": 1222,
+          "logprob": -1.75,
           "special": false,
-          "text": " test"
+          "text": " used"
         }
       ],
       "top_tokens": null
     },
-    "generated_text": "\nThe test request is a request for a test"
+    "generated_text": "\nThe test request is a document that is used"
   },
   {
     "details": {
@@ -203,68 +203,68 @@
       "tokens": [
         {
           "id": 185,
-          "logprob": -1.5546875,
+          "logprob": -1.546875,
           "special": false,
           "text": "\n"
         },
         {
           "id": 549,
-          "logprob": -2.8125,
+          "logprob": -2.859375,
           "special": false,
           "text": "The"
         },
         {
           "id": 1727,
-          "logprob": -2.375,
+          "logprob": -2.359375,
           "special": false,
           "text": " test"
         },
         {
           "id": 3102,
-          "logprob": -0.890625,
+          "logprob": -0.83203125,
           "special": false,
           "text": " request"
         },
         {
           "id": 317,
-          "logprob": -1.1484375,
+          "logprob": -1.125,
           "special": false,
           "text": " is"
         },
         {
           "id": 245,
-          "logprob": -1.5390625,
+          "logprob": -1.5703125,
           "special": false,
           "text": " a"
         },
         {
-          "id": 3102,
-          "logprob": -2.609375,
+          "id": 3412,
+          "logprob": -2.578125,
           "special": false,
-          "text": " request"
+          "text": " document"
         },
         {
-          "id": 327,
-          "logprob": -0.75,
+          "id": 344,
+          "logprob": -1.125,
           "special": false,
-          "text": " for"
+          "text": " that"
         },
         {
-          "id": 245,
-          "logprob": -1.1171875,
+          "id": 317,
+          "logprob": -1.6953125,
           "special": false,
-          "text": " a"
+          "text": " is"
         },
         {
-          "id": 1727,
-          "logprob": -0.90625,
+          "id": 1222,
+          "logprob": -1.75,
           "special": false,
-          "text": " test"
+          "text": " used"
         }
       ],
       "top_tokens": null
     },
-    "generated_text": "\nThe test request is a request for a test"
+    "generated_text": "\nThe test request is a document that is used"
   },
   {
     "details": {
@@ -292,67 +292,67 @@
       "tokens": [
         {
           "id": 185,
-          "logprob": -1.5546875,
+          "logprob": -1.546875,
           "special": false,
           "text": "\n"
         },
         {
           "id": 549,
-          "logprob": -2.8125,
+          "logprob": -2.859375,
           "special": false,
           "text": "The"
         },
         {
           "id": 1727,
-          "logprob": -2.375,
+          "logprob": -2.359375,
           "special": false,
           "text": " test"
         },
         {
           "id": 3102,
-          "logprob": -0.890625,
+          "logprob": -0.83203125,
           "special": false,
           "text": " request"
         },
         {
           "id": 317,
-          "logprob": -1.1484375,
+          "logprob": -1.125,
           "special": false,
           "text": " is"
         },
         {
           "id": 245,
-          "logprob": -1.5390625,
+          "logprob": -1.5703125,
           "special": false,
           "text": " a"
         },
         {
-          "id": 3102,
-          "logprob": -2.609375,
+          "id": 3412,
+          "logprob": -2.578125,
           "special": false,
-          "text": " request"
+          "text": " document"
         },
         {
-          "id": 327,
-          "logprob": -0.75,
+          "id": 344,
+          "logprob": -1.125,
           "special": false,
-          "text": " for"
+          "text": " that"
         },
         {
-          "id": 245,
-          "logprob": -1.1171875,
+          "id": 317,
+          "logprob": -1.6953125,
           "special": false,
-          "text": " a"
+          "text": " is"
         },
         {
-          "id": 1727,
-          "logprob": -0.90625,
+          "id": 1222,
+          "logprob": -1.75,
           "special": false,
-          "text": " test"
+          "text": " used"
         }
       ],
       "top_tokens": null
     },
-    "generated_text": "\nThe test request is a request for a test"
+    "generated_text": "\nThe test request is a document that is used"
   }
 ]
diff --git a/integration-tests/models/__snapshots__/test_flash_gemma/test_flash_gemma.json b/integration-tests/models/__snapshots__/test_flash_gemma/test_flash_gemma.json
index 80f0d053..8829f9fe 100644
--- a/integration-tests/models/__snapshots__/test_flash_gemma/test_flash_gemma.json
+++ b/integration-tests/models/__snapshots__/test_flash_gemma/test_flash_gemma.json
@@ -11,12 +11,12 @@
       },
       {
         "id": 2015,
-        "logprob": -10.0,
+        "logprob": -10.0625,
         "text": "Test"
       },
       {
         "id": 3853,
-        "logprob": -10.875,
+        "logprob": -11.0,
         "text": " request"
       }
     ],
@@ -24,7 +24,7 @@
     "tokens": [
       {
         "id": 1736,
-        "logprob": -2.09375,
+        "logprob": -2.03125,
         "special": false,
         "text": " form"
       },
@@ -42,48 +42,48 @@
       },
       {
         "id": 2121,
-        "logprob": -1.8203125,
+        "logprob": -1.8125,
         "special": false,
         "text": " test"
       },
       {
         "id": 3853,
-        "logprob": -0.23242188,
+        "logprob": -0.24121094,
         "special": false,
         "text": " request"
       },
       {
         "id": 1736,
-        "logprob": -0.08544922,
+        "logprob": -0.100097656,
         "special": false,
         "text": " form"
       },
       {
         "id": 603,
-        "logprob": -0.9375,
+        "logprob": -0.9453125,
         "special": false,
         "text": " is"
       },
       {
-        "id": 1671,
-        "logprob": -1.671875,
+        "id": 476,
+        "logprob": -1.703125,
         "special": false,
-        "text": " used"
+        "text": " a"
       },
       {
-        "id": 577,
-        "logprob": -0.40429688,
+        "id": 4551,
+        "logprob": -2.453125,
         "special": false,
-        "text": " to"
+        "text": " document"
       },
       {
-        "id": 3853,
-        "logprob": -1.1875,
+        "id": 674,
+        "logprob": -0.796875,
         "special": false,
-        "text": " request"
+        "text": " that"
       }
     ],
     "top_tokens": null
   },
-  "generated_text": " form\n\nThe test request form is used to request"
+  "generated_text": " form\n\nThe test request form is a document that"
 }
diff --git a/integration-tests/models/__snapshots__/test_flash_gemma/test_flash_gemma_all_params.json b/integration-tests/models/__snapshots__/test_flash_gemma/test_flash_gemma_all_params.json
index 8253dc96..0b840bfd 100644
--- a/integration-tests/models/__snapshots__/test_flash_gemma/test_flash_gemma_all_params.json
+++ b/integration-tests/models/__snapshots__/test_flash_gemma/test_flash_gemma_all_params.json
@@ -11,12 +11,12 @@
       },
       {
         "id": 2015,
-        "logprob": -10.0,
+        "logprob": -10.0625,
         "text": "Test"
       },
       {
         "id": 3853,
-        "logprob": -10.875,
+        "logprob": -11.0,
         "text": " request"
       }
     ],
@@ -24,7 +24,7 @@
     "tokens": [
       {
         "id": 7539,
-        "logprob": -0.73046875,
+        "logprob": -0.609375,
         "special": false,
         "text": " forms"
       },
@@ -36,7 +36,7 @@
       },
       {
         "id": 671,
-        "logprob": -1.703125,
+        "logprob": -1.5546875,
         "special": false,
         "text": " an"
       },
@@ -66,24 +66,24 @@
       },
       {
         "id": 11859,
-        "logprob": -1.6953125,
+        "logprob": -1.953125,
         "special": false,
         "text": " lab"
       },
       {
         "id": 2185,
-        "logprob": -1.3125,
+        "logprob": -1.7734375,
         "special": false,
         "text": " process"
       },
       {
-        "id": 578,
-        "logprob": -1.5,
+        "id": 235265,
+        "logprob": 0.0,
         "special": false,
-        "text": " and"
+        "text": "."
       }
     ],
     "top_tokens": null
   },
-  "generated_text": "Test request forms are an essential part of the lab process and"
+  "generated_text": "Test request forms are an essential part of the lab process."
 }
diff --git a/integration-tests/models/__snapshots__/test_flash_gemma_gptq/test_flash_gemma_gptq_all_params.json b/integration-tests/models/__snapshots__/test_flash_gemma_gptq/test_flash_gemma_gptq_all_params.json
index 7a168b2e..bc80a0f9 100644
--- a/integration-tests/models/__snapshots__/test_flash_gemma_gptq/test_flash_gemma_gptq_all_params.json
+++ b/integration-tests/models/__snapshots__/test_flash_gemma_gptq/test_flash_gemma_gptq_all_params.json
@@ -11,12 +11,12 @@
       },
       {
         "id": 2015,
-        "logprob": -9.65625,
+        "logprob": -9.640625,
         "text": "Test"
       },
       {
         "id": 3853,
-        "logprob": -10.3671875,
+        "logprob": -10.375,
         "text": " request"
       }
     ],
@@ -24,66 +24,66 @@
     "tokens": [
       {
         "id": 604,
-        "logprob": -0.36938477,
+        "logprob": -0.2824707,
         "special": false,
         "text": " for"
       },
       {
-        "id": 235248,
-        "logprob": -1.8046875,
+        "id": 573,
+        "logprob": -0.19030762,
         "special": false,
-        "text": " "
+        "text": " the"
       },
       {
-        "id": 235274,
-        "logprob": -0.46240234,
+        "id": 16819,
+        "logprob": -1.4892578,
         "special": false,
-        "text": "1"
+        "text": " detection"
       },
       {
-        "id": 235284,
-        "logprob": -1.7460938,
+        "id": 576,
+        "logprob": -0.7011719,
         "special": false,
-        "text": "2"
+        "text": " of"
       },
       {
-        "id": 235265,
-        "logprob": -1.9443359,
+        "id": 573,
+        "logprob": -2.0195312,
         "special": false,
-        "text": "."
+        "text": " the"
       },
       {
-        "id": 235284,
-        "logprob": -1.4550781,
-        "special": false,
-        "text": "2"
-      },
-      {
-        "id": 235308,
-        "logprob": -1.0205078,
-        "special": false,
-        "text": "5"
-      },
-      {
-        "id": 235290,
-        "logprob": -1.0283203,
-        "special": false,
-        "text": "-"
-      },
-      {
-        "id": 235274,
-        "logprob": -1.2783203,
-        "special": false,
-        "text": "1"
-      },
-      {
-        "id": 235284,
+        "id": 8566,
         "logprob": 0.0,
         "special": false,
-        "text": "2"
+        "text": " presence"
+      },
+      {
+        "id": 689,
+        "logprob": -0.16491699,
+        "special": false,
+        "text": " or"
+      },
+      {
+        "id": 14862,
+        "logprob": 0.0,
+        "special": false,
+        "text": " absence"
+      },
+      {
+        "id": 576,
+        "logprob": -0.9946289,
+        "special": false,
+        "text": " of"
+      },
+      {
+        "id": 671,
+        "logprob": -0.5263672,
+        "special": false,
+        "text": " an"
       }
     ],
     "top_tokens": null
   },
-  "generated_text": "Test request for 12.25-12"
+  "generated_text": "Test request for the detection of the presence or absence of an"
 }
diff --git a/integration-tests/models/__snapshots__/test_flash_starcoder/test_flash_starcoder_default_params.json b/integration-tests/models/__snapshots__/test_flash_starcoder/test_flash_starcoder_default_params.json
index 89e02c07..164e3cf2 100644
--- a/integration-tests/models/__snapshots__/test_flash_starcoder/test_flash_starcoder_default_params.json
+++ b/integration-tests/models/__snapshots__/test_flash_starcoder/test_flash_starcoder_default_params.json
@@ -11,17 +11,17 @@
       },
       {
         "id": 1459,
-        "logprob": -5.6328125,
+        "logprob": -5.625,
         "text": " print"
       },
       {
         "id": 81,
-        "logprob": -1.6035156,
+        "logprob": -1.6064453,
         "text": "_"
       },
       {
         "id": 7656,
-        "logprob": -5.9882812,
+        "logprob": -5.9921875,
         "text": "hello"
       }
     ],
@@ -29,7 +29,7 @@
     "tokens": [
       {
         "id": 2262,
-        "logprob": -0.042999268,
+        "logprob": -0.045715332,
         "special": false,
         "text": "():"
       },
@@ -59,7 +59,7 @@
       },
       {
         "id": 10896,
-        "logprob": -0.38549805,
+        "logprob": -0.3659668,
         "special": false,
         "text": " World"
       },
@@ -113,7 +113,7 @@
       },
       {
         "id": 426,
-        "logprob": 0.0,
+        "logprob": -0.051635742,
         "special": false,
         "text": "name"
       },
@@ -323,7 +323,7 @@
       },
       {
         "id": 313,
-        "logprob": -0.6328125,
+        "logprob": -0.6933594,
         "special": false,
         "text": " \""
       },
@@ -387,7 +387,8 @@
         "special": false,
         "text": " print"
       }
-    ]
+    ],
+    "top_tokens": null
   },
   "generated_text": "():\n    print(\"Hello World\")\n\ndef print_hello_name(name):\n    print(\"Hello \" + name)\n\ndef print_hello_name_age(name, age):\n    print(\"Hello \" + name + \" \" + str(age))\n\ndef print"
 }
diff --git a/integration-tests/models/__snapshots__/test_flash_starcoder2/test_flash_starcoder2_default_params.json b/integration-tests/models/__snapshots__/test_flash_starcoder2/test_flash_starcoder2_default_params.json
index 38117272..d882b82a 100644
--- a/integration-tests/models/__snapshots__/test_flash_starcoder2/test_flash_starcoder2_default_params.json
+++ b/integration-tests/models/__snapshots__/test_flash_starcoder2/test_flash_starcoder2_default_params.json
@@ -11,12 +11,12 @@
       },
       {
         "id": 1489,
-        "logprob": -5.2617188,
+        "logprob": -5.265625,
         "text": " print"
       },
       {
         "id": 100,
-        "logprob": -0.38476562,
+        "logprob": -0.38549805,
         "text": "_"
       },
       {
@@ -29,7 +29,7 @@
     "tokens": [
       {
         "id": 2284,
-        "logprob": -0.296875,
+        "logprob": -0.31323242,
         "special": false,
         "text": "():"
       },
@@ -53,19 +53,19 @@
       },
       {
         "id": 8302,
-        "logprob": -0.28125,
+        "logprob": -0.26611328,
         "special": false,
         "text": "Hello"
       },
       {
         "id": 10914,
-        "logprob": -0.79248047,
+        "logprob": -0.7817383,
         "special": false,
         "text": " World"
       },
       {
         "id": 16013,
-        "logprob": -0.61816406,
+        "logprob": -0.6328125,
         "special": false,
         "text": "!\")"
       },
@@ -83,7 +83,7 @@
       },
       {
         "id": 610,
-        "logprob": -0.4091797,
+        "logprob": -0.4086914,
         "special": false,
         "text": "def"
       },
@@ -113,7 +113,7 @@
       },
       {
         "id": 444,
-        "logprob": -0.21655273,
+        "logprob": -0.21826172,
         "special": false,
         "text": "name"
       },
@@ -160,16 +160,28 @@
         "text": "Hello"
       },
       {
-        "id": 332,
-        "logprob": -0.034698486,
+        "id": 925,
+        "logprob": -3.3476562,
         "special": false,
-        "text": " \""
+        "text": " %"
       },
       {
-        "id": 494,
+        "id": 120,
         "logprob": 0.0,
         "special": false,
-        "text": " +"
+        "text": "s"
+      },
+      {
+        "id": 11571,
+        "logprob": -0.10021973,
+        "special": false,
+        "text": "!\""
+      },
+      {
+        "id": 925,
+        "logprob": 0.0,
+        "special": false,
+        "text": " %"
       },
       {
         "id": 655,
@@ -178,22 +190,10 @@
         "text": " name"
       },
       {
-        "id": 494,
-        "logprob": -0.20141602,
-        "special": false,
-        "text": " +"
-      },
-      {
-        "id": 332,
+        "id": 46,
         "logprob": 0.0,
         "special": false,
-        "text": " \""
-      },
-      {
-        "id": 16013,
-        "logprob": 0.0,
-        "special": false,
-        "text": "!\")"
+        "text": ")"
       },
       {
         "id": 222,
@@ -251,7 +251,7 @@
       },
       {
         "id": 400,
-        "logprob": 0.0,
+        "logprob": -0.074279785,
         "special": false,
         "text": "age"
       },
@@ -310,34 +310,22 @@
         "text": "Hello"
       },
       {
-        "id": 332,
+        "id": 925,
         "logprob": 0.0,
         "special": false,
-        "text": " \""
+        "text": " %"
       },
       {
-        "id": 494,
+        "id": 120,
         "logprob": 0.0,
         "special": false,
-        "text": " +"
+        "text": "s"
       },
       {
-        "id": 655,
-        "logprob": 0.0,
+        "id": 49,
+        "logprob": -0.07891846,
         "special": false,
-        "text": " name"
-      },
-      {
-        "id": 494,
-        "logprob": 0.0,
-        "special": false,
-        "text": " +"
-      },
-      {
-        "id": 3021,
-        "logprob": -0.5761719,
-        "special": false,
-        "text": " \","
+        "text": ","
       },
       {
         "id": 863,
@@ -352,43 +340,55 @@
         "text": " are"
       },
       {
-        "id": 332,
+        "id": 925,
         "logprob": 0.0,
         "special": false,
-        "text": " \""
+        "text": " %"
       },
       {
-        "id": 494,
+        "id": 105,
         "logprob": 0.0,
         "special": false,
-        "text": " +"
+        "text": "d"
       },
       {
-        "id": 615,
+        "id": 11339,
         "logprob": 0.0,
         "special": false,
-        "text": " str"
+        "text": " years"
       },
       {
-        "id": 45,
+        "id": 3627,
         "logprob": 0.0,
         "special": false,
-        "text": "("
+        "text": " old"
       },
       {
-        "id": 400,
+        "id": 11571,
         "logprob": 0.0,
         "special": false,
-        "text": "age"
+        "text": "!\""
       },
       {
-        "id": 46,
+        "id": 925,
         "logprob": 0.0,
         "special": false,
-        "text": ")"
+        "text": " %"
+      },
+      {
+        "id": 327,
+        "logprob": 0.0,
+        "special": false,
+        "text": " ("
+      },
+      {
+        "id": 444,
+        "logprob": 0.0,
+        "special": false,
+        "text": "name"
       }
     ],
     "top_tokens": null
   },
-  "generated_text": "():\n    print(\"Hello World!\")\n\ndef print_hello_name(name):\n    print(\"Hello \" + name + \"!\")\n\ndef print_hello_name_age(name, age):\n    print(\"Hello \" + name + \", you are \" + str(age)"
+  "generated_text": "():\n    print(\"Hello World!\")\n\ndef print_hello_name(name):\n    print(\"Hello %s!\" % name)\n\ndef print_hello_name_age(name, age):\n    print(\"Hello %s, you are %d years old!\" % (name"
 }
diff --git a/integration-tests/models/__snapshots__/test_mamba/test_mamba.json b/integration-tests/models/__snapshots__/test_mamba/test_mamba.json
index eaba5078..9079b3bd 100644
--- a/integration-tests/models/__snapshots__/test_mamba/test_mamba.json
+++ b/integration-tests/models/__snapshots__/test_mamba/test_mamba.json
@@ -14,55 +14,55 @@
       },
       {
         "id": 187,
-        "logprob": -0.26953125,
+        "logprob": -0.35742188,
         "special": false,
         "text": "\n"
       },
       {
         "id": 30763,
-        "logprob": -1.1953125,
+        "logprob": -1.1015625,
         "special": false,
         "text": "Deep"
       },
       {
         "id": 4715,
-        "logprob": -0.53515625,
+        "logprob": -0.5234375,
         "special": false,
         "text": " learning"
       },
       {
         "id": 310,
-        "logprob": -0.625,
+        "logprob": -0.55078125,
         "special": false,
         "text": " is"
       },
       {
         "id": 247,
-        "logprob": -0.6796875,
+        "logprob": -0.6640625,
         "special": false,
         "text": " a"
       },
       {
         "id": 747,
-        "logprob": -2.0,
+        "logprob": -2.0625,
         "special": false,
         "text": " new"
       },
       {
         "id": 1511,
-        "logprob": -2.3125,
+        "logprob": -2.375,
         "special": false,
         "text": " type"
       },
       {
         "id": 273,
-        "logprob": -0.0028533936,
+        "logprob": -0.0029144287,
         "special": false,
         "text": " of"
       },
       {
         "id": 5145,
-        "logprob": -1.265625,
+        "logprob": -1.2734375,
         "special": false,
         "text": " machine"
       }
diff --git a/integration-tests/models/__snapshots__/test_mamba/test_mamba_all_params.json b/integration-tests/models/__snapshots__/test_mamba/test_mamba_all_params.json
index 85e9a9e0..ef88926c 100644
--- a/integration-tests/models/__snapshots__/test_mamba/test_mamba_all_params.json
+++ b/integration-tests/models/__snapshots__/test_mamba/test_mamba_all_params.json
@@ -52,7 +52,7 @@
       },
       {
         "id": 9830,
-        "logprob": -1.65625,
+        "logprob": -2.25,
         "special": false,
         "text": " colors"
       },
@@ -64,13 +64,13 @@
       },
       {
         "id": 329,
-        "logprob": -2.4375,
+        "logprob": -2.171875,
         "special": false,
         "text": " A"
       },
       {
         "id": 1180,
-        "logprob": -1.953125,
+        "logprob": -2.046875,
         "special": false,
         "text": " number"
       },